diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 064091a..acb859c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,9 +118,17 @@ jobs: # checks stay on. - name: Engine check + tests run: | - bash scripts/ci-time.sh "Engine library tests" cargo test --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance --lib --timings - bash scripts/ci-time.sh "Native runtime configuration" cargo test --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance --test native_runtime --timings - bash scripts/ci-time.sh "Engine binaries check" cargo check --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance --bins --timings + bash scripts/ci-time.sh "Engine library tests" cargo test --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance,transparent-pir --lib --timings + bash scripts/ci-time.sh "Native runtime configuration" cargo test --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance,transparent-pir --test native_runtime --timings + bash scripts/ci-time.sh "Engine binaries check" cargo check --config .github/ci/native-profile.toml -p z-engine --features native,cli,native-pir,private-enhance,transparent-pir --bins --timings + - name: Yielding transparent PIR recovery tests + # Real-server fixtures run serially within the existing two-job budget. + # Their acknowledged cold-resume and fork cases are separate from the + # engine's canonical-wallet projection tests. + timeout-minutes: 20 + env: + RAYON_NUM_THREADS: "2" + run: bash scripts/ci-time.sh "Transparent PIR recovery tests" cargo test --locked --config .github/ci/native-profile.toml -p z-transparent-pir-v2 --lib --test recovery --timings -- --test-threads=1 - name: Desktop check run: bash scripts/ci-time.sh "Desktop check" cargo check --config .github/ci/native-profile.toml -p z-desktop --timings - name: Node launcher check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35f3af3..3ef33ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,7 @@ are checked in; no separate fork checkout is needed. ```sh pnpm install --frozen-lockfile +cargo fetch --locked pnpm check:patches cargo fmt --all -- --check cargo test -p z-engine --features native --lib diff --git a/Cargo.lock b/Cargo.lock index 56b1cbf..f169eb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -562,6 +562,61 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -595,6 +650,15 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -3971,6 +4035,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4433,6 +4503,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-metal" version = "0.3.2" @@ -4740,6 +4820,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pir-control" +version = "0.1.0" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "hex", + "rand 0.8.8", + "serde", + "sha2 0.10.9", +] + [[package]] name = "pir-native" version = "0.1.0" @@ -4752,6 +4843,16 @@ dependencies = [ "reinspiring", ] +[[package]] +name = "pir-observability" +version = "0.1.0" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "axum", + "http-body", + "serde", +] + [[package]] name = "pkg-config" version = "0.3.34" @@ -6024,6 +6125,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_repr" version = "0.1.21" @@ -6601,6 +6713,20 @@ dependencies = [ "windows 0.57.0", ] +[[package]] +name = "sysinfo" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows 0.61.3", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -6665,7 +6791,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand 2.5.0", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -6854,7 +6980,9 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -7098,6 +7226,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -7116,6 +7260,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -7220,6 +7365,39 @@ dependencies = [ "transparent-filter", ] +[[package]] +name = "transparent-shard-server" +version = "0.1.0-rc.4" +source = "git+https://github.com/valargroup/wallet-pir?rev=c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2#c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" +dependencies = [ + "axum", + "base64", + "clap", + "hex", + "http-body", + "ipir-sp", + "memmap2", + "pir-control", + "pir-observability", + "rayon", + "reinspiring", + "rustix 1.1.4", + "serde", + "serde_json", + "sha2 0.10.9", + "sysinfo 0.36.1", + "thiserror 2.0.20", + "tokio", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", + "transparent-events", + "transparent-filter", + "transparent-native", + "transparent-shard", +] + [[package]] name = "transparent-wallet" version = "0.1.0-rc.4" @@ -8556,7 +8734,7 @@ dependencies = [ [[package]] name = "z-desktop" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "gpui", @@ -8570,11 +8748,12 @@ dependencies = [ [[package]] name = "z-engine" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "argon2", "async-trait", + "axum", "base64", "bip39", "blake2b_simd", @@ -8618,8 +8797,11 @@ dependencies = [ "tracing-subscriber", "transparent-events", "transparent-filter", + "transparent-shard", + "transparent-shard-server", "transparent-wallet", "z-enhance-pir", + "z-transparent-pir-v2", "zakura-address", "zakura-bls12-381", "zakura-client-backend", @@ -8641,7 +8823,7 @@ dependencies = [ [[package]] name = "z-enhance-pir" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "base64", "enhance-pir", @@ -8660,7 +8842,7 @@ dependencies = [ [[package]] name = "z-node-launcher" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "anyhow", "clap", @@ -8668,9 +8850,35 @@ dependencies = [ "serde_json", ] +[[package]] +name = "z-transparent-pir-v2" +version = "0.1.0-alpha.12" +dependencies = [ + "axum", + "base64", + "bincode", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hex", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tower", + "transparent-events", + "transparent-filter", + "transparent-native", + "transparent-shard", + "transparent-shard-server", + "transparent-wallet", +] + [[package]] name = "z-wasm" -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" dependencies = [ "console_error_panic_hook", "getrandom 0.4.3", @@ -8683,6 +8891,7 @@ dependencies = [ "wasm-bindgen-rayon", "z-engine", "z-enhance-pir", + "z-transparent-pir-v2", ] [[package]] @@ -9432,7 +9641,7 @@ dependencies = [ "rand 0.8.8", "screencapturekit", "screencapturekit-sys", - "sysinfo", + "sysinfo 0.31.4", "tao-core-video-sys", "windows 0.61.3", "windows-capture", diff --git a/Cargo.toml b/Cargo.toml index 3dc86db..f257e14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "crates/z-engine", "crates/z-enhance-pir", + "crates/z-transparent-pir-v2", "crates/z-wasm", "crates/z-node-launcher", "crates/z-desktop", @@ -11,7 +12,7 @@ members = [ exclude = ["vendor/zakura", "vendor/pir"] [workspace.package] -version = "0.1.0-alpha.11" +version = "0.1.0-alpha.12" edition = "2021" rust-version = "1.91" license = "Apache-2.0" diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index ee2d9ff..4082282 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -2115,6 +2115,135 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +base58ck 0.1.101 — CC0-1.0 +https://crates.io/crates/base58ck/0.1.101 +Published Cargo license declaration: CC0-1.0 + +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + ======================================================================== base64 0.22.1 — MIT OR Apache-2.0 https://crates.io/crates/base64/0.22.1 @@ -2379,6 +2508,34 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +bincode 1.3.3 — MIT +https://crates.io/crates/bincode/1.3.3 + +--- LICENSE.md (https://docs.rs/crate/bincode/1.3.3/source/LICENSE.md) --- +The MIT License (MIT) + +Copyright (c) 2014 Ty Overby + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== bip32 0.6.0-pre.1 — Apache-2.0 OR MIT https://crates.io/crates/bip32/0.6.0-pre.1 @@ -2741,8 +2898,8 @@ express Statement of Purpose. ======================================================================== -bitcoin_hashes 0.14.101 — CC0-1.0 -https://crates.io/crates/bitcoin_hashes/0.14.101 +bitcoin 0.32.102 — CC0-1.0 +https://crates.io/crates/bitcoin/0.32.102 Published Cargo license declaration: CC0-1.0 --- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- @@ -2870,188 +3027,575 @@ express Statement of Purpose. ======================================================================== -bitflags 2.13.1 — MIT OR Apache-2.0 -https://crates.io/crates/bitflags/2.13.1 +bitcoin-io 0.1.101 — CC0-1.0 +https://crates.io/crates/bitcoin-io/0.1.101 +Published Cargo license declaration: CC0-1.0 ---- LICENSE-APACHE (https://docs.rs/crate/bitflags/2.13.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +CC0 1.0 Universal -1. Definitions. + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +Statement of Purpose - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +4. Limitations and Disclaimers. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +======================================================================== +bitcoin-units 0.1.101 — CC0-1.0 +https://crates.io/crates/bitcoin-units/0.1.101 +Published Cargo license declaration: CC0-1.0 -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +CC0 1.0 Universal - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +Statement of Purpose - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. -END OF TERMS AND CONDITIONS +4. Limitations and Disclaimers. -APPENDIX: How to apply the Apache License to your work. + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + +======================================================================== +bitcoin_hashes 0.14.101 — CC0-1.0 +https://crates.io/crates/bitcoin_hashes/0.14.101 +Published Cargo license declaration: CC0-1.0 + +--- bitcoin_hashes-CC0.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/CC0-1.0.txt) --- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + + +======================================================================== +bitflags 2.13.1 — MIT OR Apache-2.0 +https://crates.io/crates/bitflags/2.13.1 + +--- LICENSE-APACHE (https://docs.rs/crate/bitflags/2.13.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" @@ -11005,7 +11549,1646 @@ https://crates.io/crates/env_logger/0.11.11 APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/env_logger/0.11.11/source/LICENSE-MIT) --- +Copyright (c) Individual contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +equivalent 1.0.2 — Apache-2.0 OR MIT +https://crates.io/crates/equivalent/1.0.2 + +--- LICENSE-APACHE (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-MIT) --- +Copyright (c) 2016--2023 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +f4jumble 0.1.1 — MIT OR Apache-2.0 +https://crates.io/crates/f4jumble/0.1.1 + +--- LICENSE-APACHE (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-MIT) --- +The MIT License (MIT) + +Copyright (c) 2021 Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +======================================================================== +fastrand 2.5.0 — Apache-2.0 OR MIT +https://crates.io/crates/fastrand/2.5.0 + +--- LICENSE-APACHE (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-MIT) --- +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +ff 0.14.0 — MIT/Apache-2.0 +https://crates.io/crates/ff/0.14.0 + +--- LICENSE-APACHE (https://docs.rs/crate/ff/0.14.0/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +--- LICENSE-MIT (https://docs.rs/crate/ff/0.14.0/source/LICENSE-MIT) --- +The MIT License (MIT) + +Copyright (c) 2017 Sean Bowe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +======================================================================== +find-msvc-tools 0.1.11 — MIT OR Apache-2.0 +https://crates.io/crates/find-msvc-tools/0.1.11 + +--- LICENSE-APACHE (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-MIT) --- +Copyright (c) 2014 Alex Crichton + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +fixedbitset 0.5.7 — MIT OR Apache-2.0 +https://crates.io/crates/fixedbitset/0.5.7 + +--- LICENSE-APACHE (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +--- LICENSE-MIT (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-MIT) --- +Copyright (c) 2015-2017 + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +======================================================================== +flume 0.11.1 — Apache-2.0/MIT +https://crates.io/crates/flume/0.11.1 + +--- LICENSE-APACHE (https://docs.rs/crate/flume/0.11.1/source/LICENSE-APACHE) --- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -11013,7 +13196,7 @@ https://crates.io/crates/env_logger/0.11.11 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11027,35 +13210,78 @@ https://crates.io/crates/env_logger/0.11.11 See the License for the specific language governing permissions and limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/flume/0.11.1/source/LICENSE-MIT) --- +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. ---- LICENSE-MIT (https://docs.rs/crate/env_logger/0.11.11/source/LICENSE-MIT) --- -Copyright (c) Individual contributors +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +======================================================================== +foldhash 0.1.5 — Zlib +https://crates.io/crates/foldhash/0.1.5 -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +--- LICENSE (https://docs.rs/crate/foldhash/0.1.5/source/LICENSE) --- +Copyright (c) 2024 Orson Peters -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +This software is provided 'as-is', without any express or implied warranty. In +no event will the authors be held liable for any damages arising from the use of +this software. +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim + that you wrote the original software. If you use this software in a product, + an acknowledgment in the product documentation would be appreciated but is + not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. ======================================================================== -equivalent 1.0.2 — Apache-2.0 OR MIT -https://crates.io/crates/equivalent/1.0.2 +fpe 0.6.1 — MIT/Apache-2.0 +https://crates.io/crates/fpe/0.6.1 ---- LICENSE-APACHE (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-APACHE) --- +--- COPYRIGHT (https://docs.rs/crate/fpe/0.6.1/source/COPYRIGHT) --- +Copyrights in the "fpe" library are retained by their contributors. No +copyright assignment is required to contribute to the "fpe" library. + +The "fpe" library is licensed under either of + + * Apache License, Version 2.0, (see ./LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license (see ./LICENSE-MIT or http://opensource.org/licenses/MIT) + +at your option. + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the Apache-2.0 +license, shall be dual licensed as above, without any additional terms or +conditions. + + +--- LICENSE-APACHE (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11259,9 +13485,7 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/equivalent/1.0.2/source/LICENSE-MIT) --- -Copyright (c) 2016--2023 - +--- LICENSE-MIT (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-MIT) --- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -11288,218 +13512,13 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -f4jumble 0.1.1 — MIT OR Apache-2.0 -https://crates.io/crates/f4jumble/0.1.1 - ---- LICENSE-APACHE (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - +funty 2.0.0 — MIT +https://crates.io/crates/funty/2.0.0 ---- LICENSE-MIT (https://docs.rs/crate/f4jumble/0.1.1/source/LICENSE-MIT) --- -The MIT License (MIT) +--- LICENSE.txt (https://docs.rs/crate/funty/2.0.0/source/LICENSE.txt) --- +MIT License -Copyright (c) 2021 Electric Coin Company +Copyright (c) 2020 myrrlyn (Alexander Payne) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -11508,23 +13527,23 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. ======================================================================== -fastrand 2.5.0 — Apache-2.0 OR MIT -https://crates.io/crates/fastrand/2.5.0 +futures 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11713,7 +13732,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11728,7 +13748,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fastrand/2.5.0/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -11755,10 +13778,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -ff 0.14.0 — MIT/Apache-2.0 -https://crates.io/crates/ff/0.14.0 +futures-channel 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-channel/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/ff/0.14.0/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-channel/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -11947,7 +13970,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -11962,36 +13986,40 @@ See the License for the specific language governing permissions and limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/futures-channel/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors ---- LICENSE-MIT (https://docs.rs/crate/ff/0.14.0/source/LICENSE-MIT) --- -The MIT License (MIT) - -Copyright (c) 2017 Sean Bowe - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. ======================================================================== -find-msvc-tools 0.1.11 — MIT OR Apache-2.0 -https://crates.io/crates/find-msvc-tools/0.1.11 +futures-core 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-core/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12180,7 +14208,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12195,8 +14224,9 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/find-msvc-tools/0.1.11/source/LICENSE-MIT) --- -Copyright (c) 2014 Alex Crichton +--- LICENSE-MIT (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated @@ -12224,10 +14254,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -fixedbitset 0.5.7 — MIT OR Apache-2.0 -https://crates.io/crates/fixedbitset/0.5.7 +futures-executor 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-executor/0.3.34 ---- LICENSE-APACHE (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-executor/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12416,7 +14446,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12431,242 +14462,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fixedbitset/0.5.7/source/LICENSE-MIT) --- -Copyright (c) 2015-2017 - -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - -======================================================================== -flume 0.11.1 — Apache-2.0/MIT -https://crates.io/crates/flume/0.11.1 - ---- LICENSE-APACHE (https://docs.rs/crate/flume/0.11.1/source/LICENSE-APACHE) --- - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +--- LICENSE-MIT (https://docs.rs/crate/futures-executor/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors ---- LICENSE-MIT (https://docs.rs/crate/flume/0.11.1/source/LICENSE-MIT) --- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -12691,53 +14490,12 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -======================================================================== -foldhash 0.1.5 — Zlib -https://crates.io/crates/foldhash/0.1.5 - ---- LICENSE (https://docs.rs/crate/foldhash/0.1.5/source/LICENSE) --- -Copyright (c) 2024 Orson Peters - -This software is provided 'as-is', without any express or implied warranty. In -no event will the authors be held liable for any damages arising from the use of -this software. - -Permission is granted to anyone to use this software for any purpose, including -commercial applications, and to alter it and redistribute it freely, subject to -the following restrictions: - -1. The origin of this software must not be misrepresented; you must not claim - that you wrote the original software. If you use this software in a product, - an acknowledgment in the product documentation would be appreciated but is - not required. - -2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - -3. This notice may not be removed or altered from any source distribution. ======================================================================== -fpe 0.6.1 — MIT/Apache-2.0 -https://crates.io/crates/fpe/0.6.1 - ---- COPYRIGHT (https://docs.rs/crate/fpe/0.6.1/source/COPYRIGHT) --- -Copyrights in the "fpe" library are retained by their contributors. No -copyright assignment is required to contribute to the "fpe" library. - -The "fpe" library is licensed under either of - - * Apache License, Version 2.0, (see ./LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) - * MIT license (see ./LICENSE-MIT or http://opensource.org/licenses/MIT) +futures-io 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-io/0.3.34 -at your option. - -Unless you explicitly state otherwise, any contribution intentionally -submitted for inclusion in the work by you, as defined in the Apache-2.0 -license, shall be dual licensed as above, without any additional terms or -conditions. - - ---- LICENSE-APACHE (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-io/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -12926,7 +14684,8 @@ APPENDIX: How to apply the Apache License to your work. same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12941,7 +14700,10 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/fpe/0.6.1/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures-io/0.3.34/source/LICENSE-MIT) --- +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the @@ -12968,38 +14730,10 @@ DEALINGS IN THE SOFTWARE. ======================================================================== -funty 2.0.0 — MIT -https://crates.io/crates/funty/2.0.0 +futures-macro 0.3.34 — MIT OR Apache-2.0 +https://crates.io/crates/futures-macro/0.3.34 ---- LICENSE.txt (https://docs.rs/crate/funty/2.0.0/source/LICENSE.txt) --- -MIT License - -Copyright (c) 2020 myrrlyn (Alexander Payne) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -======================================================================== -futures-core 0.3.34 — MIT OR Apache-2.0 -https://crates.io/crates/futures-core/0.3.34 - ---- LICENSE-APACHE (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-APACHE) --- +--- LICENSE-APACHE (https://docs.rs/crate/futures-macro/0.3.34/source/LICENSE-APACHE) --- Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -13204,7 +14938,7 @@ See the License for the specific language governing permissions and limitations under the License. ---- LICENSE-MIT (https://docs.rs/crate/futures-core/0.3.34/source/LICENSE-MIT) --- +--- LICENSE-MIT (https://docs.rs/crate/futures-macro/0.3.34/source/LICENSE-MIT) --- Copyright (c) 2016 Alex Crichton Copyright (c) 2017 The Tokio Authors @@ -16033,6 +17767,21 @@ express Statement of Purpose. +======================================================================== +hex_lit 0.1.1 — MITNFA +https://crates.io/crates/hex_lit/0.1.1 +Published Cargo license declaration: MITNFA + +--- hex_lit-MITNFA.txt (https://raw.githubusercontent.com/spdx/license-list-data/v3.27.0/text/MITNFA.txt) --- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +Distributions of all or part of the Software intended to be used by the recipients as they would use the unmodified Software, containing modifications that substantially alter, remove, or disable functionality of the Software, outside of the documented configuration mechanisms provided by the Software, shall be modified such that the Original Author's bug reporting email addresses and urls are either replaced with the contact information of the parties responsible for the changes, or removed entirely. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + ======================================================================== hmac 0.13.0-pre.4 — MIT OR Apache-2.0 https://crates.io/crates/hmac/0.13.0-pre.4 @@ -23202,6 +24951,35 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +pir-native 0.1.0 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/shared/pir-native +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== poly1305 0.8.0 — Apache-2.0 OR MIT https://crates.io/crates/poly1305/0.8.0 @@ -35764,6 +37542,151 @@ IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +======================================================================== +transparent-events 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-events +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-filter 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-filter +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-native 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-native +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-shard 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-shard +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +transparent-wallet 0.1.0-rc.4 — MIT +https://github.com/valargroup/wallet-pir/tree/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/transparent/crates/transparent-wallet +Published Cargo license declaration: MIT + +--- transparent-wallet-pir-LICENSE (https://raw.githubusercontent.com/valargroup/wallet-pir/c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2/LICENSE) --- +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + ======================================================================== typenum 1.20.1 — MIT OR Apache-2.0 https://crates.io/crates/typenum/1.20.1 @@ -46436,3 +48359,30 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +======================================================================== +Adapted transparent PIR asynchronous recovery walker — MIT +Copyright (c) 2025-2026 Roman Akhtariev +Original source revision: c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2 +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/crates/z-engine/Cargo.toml b/crates/z-engine/Cargo.toml index ccb49a8..010fcc3 100644 --- a/crates/z-engine/Cargo.toml +++ b/crates/z-engine/Cargo.toml @@ -59,6 +59,7 @@ lwd = [ sync = ["lwd", "zcash_client_backend/sync"] transparent-inputs = ["zcash_client_backend/transparent-inputs", "dep:transparent"] private-enhance = ["dep:z-enhance-pir"] +transparent-pir = ["transparent-inputs", "dep:z-transparent-pir-v2", "dep:transparent-events", "dep:sha2"] native-pir = ["native", "dep:transparent-wallet", "dep:transparent-events", "dep:transparent-filter", "dep:sha2", "dep:tempfile"] # WASM proving (Orchard/Ironwood in-process). Sapling params stay out of the blob. wasm-prove = ["orchard/circuit", "zcash_primitives/circuits"] @@ -96,6 +97,7 @@ cli = [ ] [dependencies] +z-transparent-pir-v2 = { path = "../z-transparent-pir-v2", optional = true } z-enhance-pir = { path = "../z-enhance-pir", optional = true } anyhow.workspace = true thiserror.workspace = true @@ -180,6 +182,10 @@ getrandom = { version = "0.2", features = ["js"] } time = { version = "0.3.22", default-features = false, features = ["std", "wasm-bindgen"] } [dev-dependencies] +axum = "0.7" +transparent-filter = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard-server = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } rand_08 = { package = "rand", version = "0.8" } hex = "0.4" enhance-pir = { path = "../../vendor/pir/enhance-pir", default-features = false } diff --git a/crates/z-engine/src/lib.rs b/crates/z-engine/src/lib.rs index 53a8652..459d5c2 100644 --- a/crates/z-engine/src/lib.rs +++ b/crates/z-engine/src/lib.rs @@ -22,6 +22,8 @@ pub mod keys; pub mod ledger; pub(crate) mod offload; pub mod scan; +#[cfg(feature = "transparent-pir")] +pub mod transparent_pir; pub mod web; mod zip321; @@ -56,6 +58,15 @@ pub enum Network { } impl Network { + /// Canonical chain identity, independent of configured endpoints. + pub fn genesis_hash(self) -> &'static str { + match self { + Self::Mainnet => "00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08", + Self::Testnet => "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38", + Self::Regtest => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", + } + } + pub fn as_str(self) -> &'static str { match self { Self::Mainnet => "mainnet", diff --git a/crates/z-engine/src/native/chain_identity.rs b/crates/z-engine/src/native/chain_identity.rs index e09bfb6..95e53cf 100644 --- a/crates/z-engine/src/native/chain_identity.rs +++ b/crates/z-engine/src/native/chain_identity.rs @@ -6,11 +6,7 @@ use crate::{ // Zakura chain parameters: checkpoint/list.rs and network/testnet.rs. pub(super) fn genesis(network: Network) -> &'static str { - match network { - Network::Mainnet => "00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08", - Network::Testnet => "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38", - Network::Regtest => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", - } + network.genesis_hash() } pub(super) fn validate(network: Network, chain: &str, hash: &str) -> Result<()> { diff --git a/crates/z-engine/src/native/mod.rs b/crates/z-engine/src/native/mod.rs index 28f00e2..7bc0907 100644 --- a/crates/z-engine/src/native/mod.rs +++ b/crates/z-engine/src/native/mod.rs @@ -108,7 +108,7 @@ pub use transparent_wallet::{ #[cfg(feature = "native-pir")] pub use wallet::{ verify_regtest_recovery_blocks, PirConfirmedTransaction, PirDiscoveryReport, PirRecoveryReport, - RecoveryCancellation, RegtestAcceptedChain, VerifiedRegtestRecoveryBlocks, + RegtestAcceptedChain, VerifiedRegtestRecoveryBlocks, }; #[cfg(feature = "native-pir")] @@ -153,3 +153,8 @@ mod replacement_tests { #[cfg(feature = "private-enhance")] pub use wallet::NativeEnhancePass; + +#[cfg(feature = "transparent-pir")] +pub use wallet::NativeTransparentPirPass; +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] +pub use wallet::RecoveryCancellation; diff --git a/crates/z-engine/src/native/wallet.rs b/crates/z-engine/src/native/wallet.rs index 9967ccb..4f7a6be 100644 --- a/crates/z-engine/src/native/wallet.rs +++ b/crates/z-engine/src/native/wallet.rs @@ -5,19 +5,25 @@ mod database_lease; mod recovery_blocks; #[cfg(feature = "native-pir")] mod recovery_bootstrap; -#[cfg(feature = "native-pir")] +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] mod recovery_cancellation; #[cfg(feature = "native-pir")] mod recovery_effects; mod recovery_guard; #[cfg(feature = "native-pir")] pub use recovery_blocks::{verify_regtest_recovery_blocks, VerifiedRegtestRecoveryBlocks}; -#[cfg(feature = "native-pir")] +#[cfg(any(feature = "native-pir", feature = "transparent-pir"))] pub use recovery_cancellation::RecoveryCancellation; mod memo_status; mod payments; mod shared_memos; +#[cfg(feature = "transparent-pir")] +mod transparent_pir; +#[cfg(feature = "transparent-pir")] +mod transparent_pir_funding; pub use shared_memos::NativeSharedMemosPass; +#[cfg(feature = "transparent-pir")] +pub use transparent_pir::NativeTransparentPirPass; #[cfg(feature = "private-enhance")] mod private_enhance; #[cfg(feature = "private-enhance")] @@ -69,7 +75,7 @@ use zcash_client_backend::{ wallet::{ create_proposed_transactions, input_selection::{GreedyInputSelector, LockFilter, SpendPolicy}, - propose_shielding, propose_transfer, ConfirmationsPolicy, SpendingKeys, + propose_transfer, ConfirmationsPolicy, SpendingKeys, }, Account, AccountBirthday, AccountPurpose, CoinbaseFilter, InputSource, MaxSpendMode, TargetValue, TransactionStatus, WalletRead, WalletWrite, @@ -3416,26 +3422,8 @@ impl NativeWallet { let (proposal, selected_generation) = db .transactionally_with_extension(|wdb, ext| -> anyhow::Result<_> { let generation = recovery_guard::transaction_recovery_generation(wdb, ext)?; - let input_selector = GreedyInputSelector::new(); - let change_strategy = SingleOutputChangeStrategy::new( - StandardFeeRule::Zip317, - None, - ShieldedPool::Orchard, - DustOutputPolicy::default(), - ); - let proposal = propose_shielding::<_, _, _, _, Infallible>( - wdb, - &self.network, - &input_selector, - &change_strategy, - threshold, - &from_addrs, - account, - shield_confirmations(self.network), - CoinbaseFilter::AllTransparentOutputs, - None, - ) - .map_err(|e| map_funds_err(format!("propose_shielding: {e}")))?; + let proposal = + self.propose_shielding_in_snapshot(wdb, ext, &from_addrs, account, threshold)?; Ok((proposal, generation)) }) .map_err(proposal_transaction_error)?; @@ -3443,6 +3431,8 @@ impl NativeWallet { let txids = db .transactionally_with_extension(|wdb, ext| -> anyhow::Result<_> { recovery_guard::transaction_proposal_binding(wdb, ext, selected_generation)?; + #[cfg(feature = "transparent-pir")] + transparent_pir_funding::check_proposal(ext, &proposal)?; let prover = crate::params::local_tx_prover()?; Ok(create_proposed_transactions::< _, @@ -3471,6 +3461,51 @@ impl NativeWallet { out } + #[cfg(feature = "transparent-inputs")] + fn propose_shielding_in_snapshot( + &self, + db: &mut D, + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, + addresses: &[transparent::address::TransparentAddress], + account: AccountUuid, + threshold: Zatoshis, + ) -> Result> + where + D: WalletWrite + + WalletRead + + InputSource::Error>, + ::Error: std::fmt::Display, + { + #[cfg(feature = "transparent-pir")] + { + transparent_pir_funding::propose(db, ext, self.network, addresses, account, threshold) + } + #[cfg(not(feature = "transparent-pir"))] + { + let _ = ext; + let input_selector = GreedyInputSelector::new(); + let change_strategy = SingleOutputChangeStrategy::new( + StandardFeeRule::Zip317, + None, + ShieldedPool::Orchard, + DustOutputPolicy::default(), + ); + zcash_client_backend::data_api::wallet::propose_shielding::<_, _, _, _, Infallible>( + db, + &self.network, + &input_selector, + &change_strategy, + threshold, + addresses, + account, + shield_confirmations(self.network), + CoinbaseFilter::AllTransparentOutputs, + None, + ) + .map_err(|e| map_funds_err(format!("propose_shielding: {e}"))) + } + } + /// After sync: auto-shield t→Orchard, then migrate Sapling→Orchard when funded. pub async fn maintain(&self, auth: &SeedAuth) -> Result> { self.require_network_access()?; @@ -3621,7 +3656,7 @@ impl NativeWallet { db.transactionally_with_extension(|wdb, ext| -> anyhow::Result { recovery_guard::transaction_recovery_ready(wdb, ext)?; after_guard(); - self.propose_fee_in_snapshot(wdb, request, policy) + self.propose_fee_in_snapshot(wdb, ext, request, policy) .map_err(anyhow::Error::new) }) .map_err(proposal_transaction_error) @@ -3630,6 +3665,7 @@ impl NativeWallet { fn propose_fee_in_snapshot( &self, db: &mut D, + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, request: TransactionRequest, policy: SpendPolicy, ) -> Result @@ -3661,12 +3697,17 @@ impl NativeWallet { None, ) .map_err(|e| map_funds_err(format!("propose_transfer: {e}")))?; + #[cfg(feature = "transparent-pir")] + transparent_pir_funding::check_proposal(ext, &proposal).map_err(EngineError::from)?; + #[cfg(not(feature = "transparent-pir"))] + let _ = ext; Ok(crate::proposal_fee_zat(&proposal)) } fn propose_amount( &self, db: &mut D, + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, dest: &str, amount_zat: u64, policy: SpendPolicy, @@ -3681,7 +3722,7 @@ impl NativeWallet { let payment = zip_payment(dest, amount_zat, None)?; let request = TransactionRequest::new(vec![payment]) .map_err(|e| EngineError::Message(format!("ZIP-321: {e}")))?; - self.propose_fee_in_snapshot(db, request, policy) + self.propose_fee_in_snapshot(db, ext, request, policy) } fn max_send_amount( @@ -3734,12 +3775,12 @@ impl NativeWallet { if amt == 0 { return Ok((0, 0)); } - match self.propose_amount(db, dest, amt, policy.clone()) { + match self.propose_amount(db, ext, dest, amt, policy.clone()) { Ok(fee) => { let exact = available.saturating_sub(fee); if exact > 0 && exact != amt { if let Ok(fee2) = - self.propose_amount(db, dest, exact, policy.clone()) + self.propose_amount(db, ext, dest, exact, policy.clone()) { return Ok((exact, fee2)); } @@ -3938,6 +3979,8 @@ impl NativeWallet { None, ) .map_err(|e| map_funds_err(format!("propose_transfer: {e}")))?; + #[cfg(feature = "transparent-pir")] + transparent_pir_funding::check_proposal(ext, &proposal)?; Ok((proposal, generation)) }) .map_err(proposal_transaction_error)?; @@ -3947,6 +3990,8 @@ impl NativeWallet { let txids = db .transactionally_with_extension(|wdb, ext| -> anyhow::Result<_> { recovery_guard::transaction_proposal_binding(wdb, ext, selected_generation)?; + #[cfg(feature = "transparent-pir")] + transparent_pir_funding::check_proposal(ext, &proposal)?; let prover = crate::params::local_tx_prover()?; Ok(create_proposed_transactions::< _, diff --git a/crates/z-engine/src/native/wallet/payments.rs b/crates/z-engine/src/native/wallet/payments.rs index 3cde85d..20dd064 100644 --- a/crates/z-engine/src/native/wallet/payments.rs +++ b/crates/z-engine/src/native/wallet/payments.rs @@ -774,6 +774,8 @@ impl NativeWallet { let mut db = WalletDb::from_connection(conn, self.network, SystemClock, new_rng()); db.transactionally_with_extension(|wdb,ext|->anyhow::Result<()> { super::recovery_guard::transaction_recovery_ready(wdb,ext)?; + #[cfg(feature = "transparent-pir")] + super::transparent_pir_funding::check_proposal(ext, &proposal)?; let marker:bool=ext.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='ext_coffer_pir_recovery_v1')",[],|r|r.get(0))?; let generation = if marker {Some(ext.query_row("SELECT generation FROM ext_coffer_pir_recovery_v1 WHERE id=1",[],|r|r.get::<_,u64>(0))?)} else {None}; if generation!=recovery_generation || wdb.chain_height()?!=Some(height) || wdb.get_block_hash(height)?.is_none_or(|h|h!=hash) {anyhow::bail!("payment_review_expired");} @@ -858,6 +860,8 @@ impl NativeWallet { } let proposal = proto.try_into_standard_proposal(&self.network, wdb) .map_err(|_| anyhow::anyhow!("payment_review_expired"))?; + #[cfg(feature = "transparent-pir")] + super::transparent_pir_funding::check_proposal(ext, &proposal)?; if crate::proposal_fee_zat(&proposal) != receipt.fee_zat { anyhow::bail!("payment_storage_failed"); } let txids = create_proposed_transactions::<_, _, Infallible, _, Infallible, _>( wdb, &self.network, &*prover, &*prover, diff --git a/crates/z-engine/src/native/wallet/pir_store.rs b/crates/z-engine/src/native/wallet/pir_store.rs index ff63189..6b1c015 100644 --- a/crates/z-engine/src/native/wallet/pir_store.rs +++ b/crates/z-engine/src/native/wallet/pir_store.rs @@ -773,7 +773,19 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let wallet = fixture_wallet(dir.path()); let (key, birthday) = fixture_account(); - wallet.replace_scan_db(&key, &birthday).unwrap(); + // This isolated store fixture needs an initial account, not a rescan. + // Avoid the reset lease hand-off while parallel tests spawn processes. + let mut db = open_wallet_db(&wallet.paths.data_db, wallet.network).unwrap(); + init_wallet_db(&mut db, None).unwrap(); + db.import_account_ufvk( + "primary", + &key, + &birthday, + AccountPurpose::Spending { derivation: None }, + Some("z-stack-reset"), + ) + .unwrap(); + drop(db); (dir, wallet) } fn commit() -> ShardCommit { diff --git a/crates/z-engine/src/native/wallet/source_wallet_tests.rs b/crates/z-engine/src/native/wallet/source_wallet_tests.rs index 67611f1..8f6e1db 100644 --- a/crates/z-engine/src/native/wallet/source_wallet_tests.rs +++ b/crates/z-engine/src/native/wallet/source_wallet_tests.rs @@ -91,6 +91,9 @@ impl Drop for Fixture { } impl Fixture { async fn new(root: &Path) -> Self { + Self::new_with_tip(root, 2).await + } + async fn new_with_tip(root: &Path, tip: u64) -> Self { let calls = Arc::new(Mutex::new(Vec::new())); let sent = Arc::new(Mutex::new(Vec::new())); let chain = Arc::new(Mutex::new("regtest".to_owned())); @@ -143,11 +146,23 @@ impl Fixture { .unwrap() .to_bytes(); let payload = &raw[5..]; + // Mode 4 is an independently selected synthetic fork for host reorg tests. + let compact = |height| { + let mut result = block(height, &g.lock().unwrap()); + if m.load(Ordering::Acquire) == 4 && height >= 2 { + result.hash = vec![(height as u8).wrapping_add(0x80); 32]; + if height > 2 { + result.prev_hash = + vec![(height as u8 - 1).wrapping_add(0x80); 32]; + } + } + result + }; let response = match method.as_str() { "GetLightdInfo" => reply( vec![LightdInfo { chain_name: n.lock().unwrap().clone(), - block_height: 2, + block_height: tip, ..Default::default() } .encode_to_vec()], @@ -155,18 +170,15 @@ impl Fixture { ), "GetLatestBlock" => reply( vec![BlockId { - height: 2, - hash: vec![2; 32], + height: tip, + hash: compact(tip).hash, } .encode_to_vec()], "0", ), "GetBlock" => reply( - vec![block( - BlockId::decode(payload).unwrap().height, - &g.lock().unwrap(), - ) - .encode_to_vec()], + vec![compact(BlockId::decode(payload).unwrap().height) + .encode_to_vec()], "0", ), "GetTreeState" => { @@ -178,7 +190,18 @@ impl Fixture { hash: if id.height == 0 { g.lock().unwrap().clone() } else { - format!("{:02x}", id.height as u8).repeat(32) + let height = id.height as u8; + format!( + "{:02x}", + if m.load(Ordering::Acquire) == 4 + && id.height >= 2 + { + height.wrapping_add(0x80) + } else { + height + } + ) + .repeat(32) }, time: 1, sapling_tree: "000000".into(), @@ -193,7 +216,7 @@ impl Fixture { let range = BlockRange::decode(payload).unwrap(); reply( (range.start.unwrap().height..=range.end.unwrap().height) - .map(|h| block(h, &g.lock().unwrap()).encode_to_vec()) + .map(|h| compact(h).encode_to_vec()) .collect(), "0", ) @@ -604,3 +627,11 @@ async fn source_rejection_keeps_saved_intent_and_exposes_only_safe_error_text() assert_eq!(wallet.payment_receipt(&id).unwrap().phase, "unknown"); assert_eq!(*fixture.sent.lock().unwrap(), vec![raw]); } + +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir_source_tests.rs"] +mod transparent_pir_projection_tests; + +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir_integration_tests.rs"] +mod transparent_pir_integration_tests; diff --git a/crates/z-engine/src/native/wallet/transparent_pir.rs b/crates/z-engine/src/native/wallet/transparent_pir.rs new file mode 100644 index 0000000..59d71ef --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir.rs @@ -0,0 +1,785 @@ +//! Source-bound canonical transparent application. No address queries or raw-transaction fabrication. +use super::*; +use crate::transparent_pir::{ + parse, refused, ProjectionState, TransparentPirAnchor, TransparentPirCheckpoint, + TransparentPirScope, TransparentPirScript, +}; +use rusqlite::OptionalExtension; +use transparent::{ + address::Script, + bundle::{OutPoint, TxOut}, +}; +use transparent_events::TransparentEvent; +use zcash_client_backend::{ + data_api::ll::{LowLevelWalletRead, LowLevelWalletWrite}, + wallet::{WalletTransparentOutput, WalletTx}, +}; + +pub struct NativeTransparentPirPass { + root: PathBuf, + binding: [u8; 32], + scope: TransparentPirScope, + cancel: Arc, + _owner: std::fs::File, +} + +#[cfg(test)] +mod inventory_tests { + use super::*; + use zcash_client_backend::data_api::WalletWrite; + + #[test] + fn native_exposed_ephemeral_overlap_counts_unique_scope_receivers() { + let root = tempfile::tempdir().unwrap(); + let wallet = crate::native::wallet::tests::fixture_wallet(root.path()); + let (key, birthday) = crate::native::wallet::tests::fixture_account(); + wallet.replace_scan_db(&key, &birthday).unwrap(); + let mut db = wallet.open_db().unwrap(); + db.update_chain_tip(1.into()).unwrap(); + drop(db); + + // Retain actual Rust-derived public receivers in the pinned SQLite row + // shape. This fixture has no spends, network transport or user wallet. + let ivk = key.transparent().unwrap().derive_ephemeral_ivk().unwrap(); + let mut conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let (account, flags): (i64, i64) = conn + .query_row( + "SELECT account_id,receiver_flags FROM addresses WHERE key_scope=2 LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + let tx = conn.transaction().unwrap(); + for index in 0..2049u32 { + let child = transparent::keys::NonHardenedChildIndex::from_index(index).unwrap(); + let address = ivk.derive_ephemeral_address(child).unwrap(); + let encoded = + zcash_keys::address::Address::Transparent(address).encode(&ZNetwork::Regtest); + let mut diversifier = *zip32::DiversifierIndex::from(index).as_bytes(); + diversifier.reverse(); + tx.execute("INSERT OR IGNORE INTO addresses(account_id,diversifier_index_be,key_scope,address,transparent_child_index,cached_transparent_receiver_address,receiver_flags,exposed_at_height) VALUES(?1,?2,2,?3,?4,?3,?5,1)", + rusqlite::params![account,diversifier.as_slice(),encoded,index,flags]).unwrap(); + } + tx.execute( + "UPDATE addresses SET exposed_at_height=1 WHERE key_scope=2", + [], + ) + .unwrap(); + tx.commit().unwrap(); + drop(conn); + + let db = wallet.open_db().unwrap(); + let account = db.get_account_ids().unwrap()[0]; + assert_eq!( + db.get_ephemeral_transparent_receivers(account, u32::MAX, false) + .unwrap() + .len(), + 2049 + ); + drop(db); + let inventory = wallet + .transparent_transaction(|db, tx| Ok(scripts(db, tx, ZNetwork::Regtest)?)) + .unwrap(); + let unique = inventory + .iter() + .map(|script| script.script.clone()) + .collect::>() + .len(); + assert!(inventory.len() > 4096); + assert!((2049..=4096).contains(&unique)); + let scope = ProjectionState::default() + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }, + inventory, + ) + .unwrap(); + assert_eq!(scope.scripts.len(), unique); + } +} +impl NativeTransparentPirPass { + pub fn scope_json(&self) -> Result { + serde_json::to_string(&self.scope).map_err(|_| refused("transparent_pir_scope_invalid")) + } + pub fn cancel(&self) { + self.cancel.cancel(); + } +} +fn database(_: impl std::fmt::Display) -> EngineError { + refused("transparent_pir_database_error") +} +// The native marker distinguishes old projection-only state from funding provenance. +// Migration conservatively quarantines prior receive identities, without changing facts, +// continuation, deletion ownership, reservations, or unknown submissions. +fn decode_native_state(bytes: &[u8]) -> anyhow::Result<(ProjectionState, bool)> { + let value: serde_json::Value = serde_json::from_slice(bytes) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let mut state: ProjectionState = serde_json::from_value(value.clone()) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let legacy = match value.get("nativeFundingVersion") { + Some(version) if version.as_u64() == Some(2) && state.unverified_outputs.is_some() => false, + None => true, + Some(version) if version.as_u64() == Some(1) => true, + _ => anyhow::bail!("transparent_pir_state_invalid"), + }; + if legacy { + state.unverified_outputs.get_or_insert_default().extend( + state + .records + .keys() + .filter(|k| k.starts_with("r:")) + .cloned(), + ); + } + state + .unverified_outputs + .get_or_insert_default() + .extend(state.projected_outputs.iter().cloned()); + let unverified = state.unverified_outputs.as_ref().unwrap(); + if unverified.len() > 65_536 { + anyhow::bail!("transparent_pir_state_invalid"); + } + for key in unverified { + let fields: Vec<_> = key.split(':').collect(); + if !matches!(fields.as_slice(), ["r", id, n] if crate::transparent_pir::hash(id) && n.parse::().is_ok_and(|v|v.to_string()==*n)) + { + anyhow::bail!("transparent_pir_state_invalid"); + } + } + Ok((state, legacy)) +} +// Native v1 and older did not reliably distinguish shared publisher-supplied +// confirmation metadata. Migrate once under the native owner; do not infer it. +const LEGACY_OUTPUTS: &str = "SELECT json_group_array(json_array(txid,output_index)) FROM (SELECT lower(hex(t.txid)) AS txid,n.output_index FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id LIMIT 65537)"; +fn quarantine_legacy(state: &mut ProjectionState, rows: &str) -> anyhow::Result<()> { + let rows: Vec<(String, u32)> = + serde_json::from_str(rows).map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + if rows.len() > 65_536 || rows.iter().any(|(id, _)| !crate::transparent_pir::hash(id)) { + anyhow::bail!("transparent_pir_state_invalid"); + } + state + .unverified_outputs + .get_or_insert_default() + .extend(rows.into_iter().map(|(id, n)| format!("r:{id}:{n}"))); + Ok(()) +} +fn encode_native_state(state: &ProjectionState) -> anyhow::Result> { + let mut value = serde_json::to_value(state)?; + if state + .unverified_outputs + .as_ref() + .is_some_and(|s| s.len() > 65_536) + { + anyhow::bail!("transparent_pir_state_invalid"); + } + value["nativeFundingVersion"] = 2.into(); + if state.unverified_outputs.is_none() { + value["unverifiedOutputs"] = serde_json::json!([]); + } + Ok(serde_json::to_vec(&value)?) +} +fn load(ext: &rusqlite::Transaction<'_>) -> anyhow::Result { + ext.execute("CREATE TABLE IF NOT EXISTS ext_transparent_pir_v1 (id INTEGER PRIMARY KEY CHECK(id=1), state BLOB NOT NULL)", [])?; + ext.execute("CREATE TABLE IF NOT EXISTS ext_transparent_pir_funding_guard_v1 (id INTEGER PRIMARY KEY CHECK(id=1))", [])?; + let required: bool = ext.query_row( + "SELECT EXISTS(SELECT 1 FROM ext_transparent_pir_funding_guard_v1)", + [], + |r| r.get(0), + )?; + let bytes: Option> = ext + .query_row( + "SELECT state FROM ext_transparent_pir_v1 WHERE id=1", + [], + |r| r.get(0), + ) + .optional()?; + match bytes { + Some(bytes) => { + let (mut state, legacy) = decode_native_state(&bytes)?; + if legacy { + let rows: String = ext.query_row(LEGACY_OUTPUTS, [], |r| r.get(0))?; + quarantine_legacy(&mut state, &rows)?; + save(ext, &state)?; + } + if state.bound_source.is_some() { + ext.execute( + "INSERT OR IGNORE INTO ext_transparent_pir_funding_guard_v1 VALUES(1)", + [], + )?; + } + Ok(state) + } + None if required => anyhow::bail!("transparent_pir_state_invalid"), + None => Ok(ProjectionState::default()), + } +} +fn save(ext: &rusqlite::Transaction<'_>, state: &ProjectionState) -> anyhow::Result<()> { + ext.execute("INSERT INTO ext_transparent_pir_v1 VALUES(1,?1) ON CONFLICT(id) DO UPDATE SET state=excluded.state", [encode_native_state(state)?])?; + if state.bound_source.is_some() { + ext.execute( + "INSERT OR IGNORE INTO ext_transparent_pir_funding_guard_v1 VALUES(1)", + [], + )?; + } + Ok(()) +} +pub(super) fn funding_exclusions( + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, +) -> anyhow::Result> { + let exists: bool = ext.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='ext_transparent_pir_v1')", [], |r|r.get(0))?; + let guard_exists: bool = ext.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='ext_transparent_pir_funding_guard_v1')", [], |r|r.get(0))?; + let required = guard_exists + && ext.query_row( + "SELECT EXISTS(SELECT 1 FROM ext_transparent_pir_funding_guard_v1)", + [], + |r| r.get::<_, bool>(0), + )?; + let bytes: Option> = if exists { + ext.query_row( + "SELECT state FROM ext_transparent_pir_v1 WHERE id=1", + [], + |r| r.get(0), + ) + .optional()? + } else { + None + }; + match bytes { + Some(bytes) => { + let (mut state, legacy) = decode_native_state(&bytes)?; + if state.bound_source.is_some() { + // DDL is intentionally unavailable to extension transactions. Old + // wallets without the guard must first load their PIR state through + // the native owner; funding remains refused in the meantime. + if !guard_exists { + anyhow::bail!("transparent_pir_state_invalid"); + } + ext.execute( + "INSERT OR IGNORE INTO ext_transparent_pir_funding_guard_v1 VALUES(1)", + [], + )?; + } + if legacy { + let rows: String = ext.query_row(LEGACY_OUTPUTS, [], |r| r.get(0))?; + quarantine_legacy(&mut state, &rows)?; + } + // Upgrade before selection in the same wallet transaction. + ext.execute( + "UPDATE ext_transparent_pir_v1 SET state=?1 WHERE id=1", + [encode_native_state(&state)?], + )?; + Ok(state.unverified_outputs.unwrap_or_default()) + } + None if required => anyhow::bail!("transparent_pir_state_invalid"), + None => Ok(Default::default()), + } +} +fn scripts>( + db: &D, + tx: &rusqlite::Transaction<'_>, + network: ZNetwork, +) -> Result> +where + D::Error: std::fmt::Display, +{ + let mut result = Vec::new(); + for account in db.get_account_ids().map_err(database)? { + let mut receivers = db + .get_transparent_receivers(account, true, true) + .map_err(database)?; + if db.chain_height().map_err(database)?.is_some() { + receivers.extend( + db.get_ephemeral_transparent_receivers(account, u32::MAX, false) + .map_err(database)?, + ); + } + for (address, _metadata) in receivers { + let script: Script = address.script().into(); + result.push(TransparentPirScript { + script: crate::web::to_hex(&script.0 .0), + required_from: 0, + }); + } + } + // Pinned WalletRead exposes only exposed ephemeral receivers. Include the + // engine's already allocated ephemeral gap without deriving outside Rust. + let mut stmt=tx.prepare("SELECT cached_transparent_receiver_address FROM addresses WHERE key_scope=2 AND cached_transparent_receiver_address IS NOT NULL ORDER BY id LIMIT 4097").map_err(database)?; + for address in stmt + .query_map([], |r| r.get::<_, String>(0)) + .map_err(database)? + { + let address = address.map_err(database)?; + let address = zcash_keys::address::Address::decode(&network, &address) + .and_then(|a| a.to_transparent_address()) + .ok_or_else(|| refused("transparent_pir_scope_invalid"))?; + let script: Script = address.script().into(); + result.push(TransparentPirScript { + script: crate::web::to_hex(&script.0 .0), + required_from: 0, + }); + if result.len() > 8192 { + return Err(refused("transparent_pir_scope_limit")); + } + } + Ok(result) +} +/// Canonical truncation can erase a projection even after scanning returns to +/// the same hash. Check only this source's owned effects in bounded SQL batches. +fn missing_projection( + tx: &rusqlite::Transaction<'_>, + state: &ProjectionState, +) -> anyhow::Result> { + use std::collections::{BTreeMap, BTreeSet}; + let mut expected = Vec::new(); + let mut ids = BTreeSet::new(); + if state + .projected_outputs + .len() + .saturating_add(state.projected_spends.len()) + > 65_536 + { + anyhow::bail!("transparent_pir_state_invalid"); + } + for key in state + .projected_outputs + .iter() + .chain(&state.projected_spends) + { + let fields: Vec<_> = key.split(':').collect(); + let height = state + .records + .get(key) + .ok_or_else(|| anyhow::anyhow!("transparent_pir_state_invalid"))? + .0; + if !matches!(fields.as_slice(), ["r", _, _] | ["s", _, _, _, _]) { + anyhow::bail!("transparent_pir_state_invalid"); + } + let id = crate::web::from_hex(fields[1]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + ids.insert(id.clone()); + let output = fields.last().unwrap().parse::()?; + let previous = if fields[0] == "s" { + Some( + crate::web::from_hex(fields[3]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?, + ) + } else { + None + }; + expected.push((key.clone(), id, height, output, previous)); + } + let ids: Vec<_> = ids.into_iter().collect(); + let mut mined = BTreeMap::new(); + let mut outputs = BTreeSet::new(); + let mut spends = BTreeSet::new(); + for batch in ids.chunks(256) { + let parameters = std::iter::repeat_n("?", batch.len()) + .collect::>() + .join(","); + let mut stmt = tx.prepare(&format!( + "SELECT txid,mined_height,tx_index FROM transactions WHERE txid IN({parameters})" + ))?; + for row in stmt.query_map(rusqlite::params_from_iter(batch), |r| { + Ok(( + r.get::<_, Vec>(0)?, + r.get::<_, Option>(1)?, + r.get::<_, Option>(2)?, + )) + })? { + let (id, height, index) = row?; + mined.insert(id, (height, index)); + } + let mut stmt = tx.prepare(&format!("SELECT t.txid,n.output_index FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid IN({parameters})"))?; + for row in stmt.query_map(rusqlite::params_from_iter(batch), |r| { + Ok((r.get::<_, Vec>(0)?, r.get::<_, u32>(1)?)) + })? { + outputs.insert(row?); + } + // A completed source projection must have its actual UTXO spend link. + // A pending map alone is sufficient only for an explicitly unresolved input. + let pending_allowed = u8::from(!state.complete && state.unresolved_spends > 0); + let mut stmt = tx.prepare(&format!("SELECT t.txid,p.txid,n.output_index FROM transparent_received_output_spends s JOIN transactions t ON t.id_tx=s.transaction_id JOIN transparent_received_outputs n ON n.id=s.transparent_received_output_id JOIN transactions p ON p.id_tx=n.transaction_id WHERE t.txid IN({parameters}) UNION SELECT t.txid,s.prevout_txid,s.prevout_output_index FROM transparent_spend_map s JOIN transactions t ON t.id_tx=s.spending_transaction_id WHERE t.txid IN({parameters}) AND {pending_allowed}=1 AND NOT EXISTS(SELECT 1 FROM transparent_received_outputs n JOIN transactions p ON p.id_tx=n.transaction_id WHERE p.txid=s.prevout_txid AND n.output_index=s.prevout_output_index)"))?; + for row in stmt.query_map(rusqlite::params_from_iter(batch.iter().chain(batch)), |r| { + Ok(( + r.get::<_, Vec>(0)?, + r.get::<_, Vec>(1)?, + r.get::<_, u32>(2)?, + )) + })? { + spends.insert(row?); + } + } + Ok(expected + .into_iter() + .filter_map(|(key, id, height, index, previous)| { + let present = mined + .get(&id) + .is_some_and(|(mined, position)| *mined == Some(height) && position.is_some()) + && match previous { + Some(previous) => spends.contains(&(id, previous, index)), + None => outputs.contains(&(id, index)), + }; + (!present).then_some((key, height)) + }) + .collect()) +} +fn invalidate_missing( + state: &mut ProjectionState, + missing: &std::collections::BTreeMap, +) -> bool { + let Some(floor) = missing + .values() + .copied() + .map(|height| height.saturating_sub(1)) + .min() + else { + return false; + }; + let changed = !state.projection_invalidated + || state.complete + || state.rewind_floor.is_none_or(|old| floor < old); + state.rewind(floor); + changed +} + +fn retire_projection( + tx: &rusqlite::Transaction<'_>, + state: &mut ProjectionState, + old: &std::collections::BTreeMap, + incoming: &[(Vec, TransparentEvent)], + height: u32, +) -> anyhow::Result<()> { + let incoming: std::collections::BTreeMap<_, _> = incoming + .iter() + .map(|(_, e)| { + ( + crate::transparent_pir::event_key(*e), + crate::transparent_pir::digest(&e.to_bytes()), + ) + }) + .collect(); + let retired: Vec<_> = old + .iter() + .filter(|(key, (h, d))| *h > height && incoming.get(*key) != Some(d)) + .map(|(k, _)| k.clone()) + .collect(); + // Spends retire before outputs. Only relations this source originally created + // are eligible; existing local pending/independent relations are never owned. + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"s") && state.projected_spends.remove(key) { + let spending = crate::web::from_hex(fields[1]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let previous = crate::web::from_hex(fields[3]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let index: u32 = fields[4].parse()?; + let local: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM transactions WHERE txid=?1 AND created IS NOT NULL)", + [&spending], + |r| r.get(0), + )?; + if local { + anyhow::bail!("transparent_pir_pending_reconciliation_required"); + } + tx.execute("DELETE FROM transparent_received_output_spends WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1) AND transparent_received_output_id IN(SELECT n.id FROM transparent_received_outputs n JOIN transactions p ON p.id_tx=n.transaction_id WHERE p.txid=?2 AND n.output_index=?3)",rusqlite::params![spending,previous,index])?; + tx.execute("DELETE FROM transparent_spend_map WHERE spending_transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1) AND prevout_txid=?2 AND prevout_output_index=?3",rusqlite::params![spending,previous,index])?; + } + } + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"r") && state.projected_outputs.remove(key) { + let id = crate::web::from_hex(fields[1]) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + let index: u32 = fields[2].parse()?; + let locked:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1 AND n.output_index=?2 AND (n.lock_owner IS NOT NULL OR n.lock_expiry_height IS NOT NULL OR EXISTS(SELECT 1 FROM transparent_received_output_spends s JOIN transactions p ON p.id_tx=s.transaction_id WHERE s.transparent_received_output_id=n.id AND p.created IS NOT NULL)))",rusqlite::params![id,index],|r|r.get(0))?; + if locked { + anyhow::bail!("transparent_pir_pending_reconciliation_required"); + } + // Retained full bytes are independent wallet evidence; leave those rows. + tx.execute("DELETE FROM transparent_received_outputs WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1 AND raw IS NULL AND created IS NULL) AND output_index=?2",rusqlite::params![id,index])?; + } + } + let ids: Vec<_> = state.projected_transactions.iter().cloned().collect(); + for id in ids { + let bytes = crate::web::from_hex(&id) + .map_err(|_| anyhow::anyhow!("transparent_pir_state_invalid"))?; + // Raw/locally-created transactions retain their independent pending authority. + tx.execute("UPDATE transactions SET mined_height=NULL,block=NULL,tx_index=NULL WHERE txid=?1 AND mined_height>?2 AND raw IS NULL AND created IS NULL",rusqlite::params![bytes,height])?; + } + Ok(()) +} + +struct TransactionConnection<'a, 'b>(&'a rusqlite::Transaction<'b>); +impl std::borrow::Borrow for TransactionConnection<'_, '_> { + fn borrow(&self) -> &rusqlite::Connection { + self.0 + } +} +impl<'a, 'b> std::borrow::Borrow> for TransactionConnection<'a, 'b> { + fn borrow(&self) -> &rusqlite::Transaction<'b> { + self.0 + } +} +type ProjectionDb<'a, 'b> = + WalletDb, ZNetwork, SystemClock, UnwrapErr>; + +impl NativeWallet { + fn transparent_transaction( + &self, + operation: impl FnOnce( + &mut ProjectionDb<'_, '_>, + &rusqlite::Transaction<'_>, + ) -> anyhow::Result, + ) -> Result { + // This engine-owned connection wraps one canonical wallet transaction. + // It is never held over transport and never exposed to the host. + let mut conn = rusqlite::Connection::open(&self.paths.data_db).map_err(database)?; + rusqlite::vtab::array::load_module(&conn).map_err(database)?; + conn.busy_timeout(Duration::from_secs(5)) + .map_err(database)?; + conn.pragma_update(None, "foreign_keys", "ON") + .map_err(database)?; + conn.pragma_update(None, "synchronous", "FULL") + .map_err(database)?; + let tx = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .map_err(database)?; + let mut wdb = WalletDb::from_connection( + TransactionConnection(&tx), + self.network, + SystemClock, + new_rng(), + ); + let result = operation(&mut wdb, &tx).map_err(EngineError::from)?; + drop(wdb); + tx.commit().map_err(database)?; + Ok(result) + } + + pub fn begin_transparent_pir_with_source( + &self, + source: &super::super::NativeWalletSource, + source_id: &str, + cancel: Arc, + ) -> Result { + self.require_wallet_source(source)?; + let owner = database_lease::shared(&self.paths.data_db)?; + let scope = cancel.transaction(|| { + self.transparent_transaction(|wdb, ext| -> anyhow::Result<_> { + let block = wdb + .block_fully_scanned()? + .ok_or_else(|| anyhow::anyhow!("transparent_pir_anchor_unavailable"))?; + let anchor = TransparentPirAnchor { + height: u64::from(u32::from(block.block_height())), + hash: block.block_hash().to_string(), + }; + let mut state = load(ext)?; + let mut scope = state.begin(source_id, anchor, scripts(wdb, ext, self.network)?)?; + let missing = missing_projection(ext, &state)?; + invalidate_missing(&mut state, &missing); + scope.accepted_headers.retain(|h| h.height != 0); + scope.accepted_headers.push(TransparentPirAnchor { + height: 0, + hash: self.network.genesis_hash().into(), + }); + state.scope = Some(scope.clone()); + save(ext, &state)?; + Ok(scope) + }) + })?; + Ok(NativeTransparentPirPass { + root: self.paths.root.clone(), + binding: source.binding(), + scope, + cancel, + _owner: owner, + }) + } + + pub fn apply_transparent_pir_checkpoint_with_source( + &self, + source: &super::super::NativeWalletSource, + pass: &mut NativeTransparentPirPass, + json: &str, + ) -> Result { + self.apply_transparent_pir_checkpoint_inner(source, pass, json, None) + } + + pub fn apply_transparent_pir_checkpoint_with_source_and_rewind( + &self, + source: &super::super::NativeWalletSource, + pass: &mut NativeTransparentPirPass, + json: &str, + rewind: z_transparent_pir_v2::AcceptedRewind<'_>, + ) -> Result { + self.apply_transparent_pir_checkpoint_inner(source, pass, json, Some(rewind)) + } + + fn apply_transparent_pir_checkpoint_inner( + &self, + source: &super::super::NativeWalletSource, + pass: &mut NativeTransparentPirPass, + json: &str, + rewind: Option>, + ) -> Result { + self.require_wallet_source(source)?; + if pass.root != self.paths.root || pass.binding != source.binding() { + return Err(refused("transparent_pir_context_changed")); + } + let checkpoint: TransparentPirCheckpoint = parse(json)?; + if checkpoint.scope_id != pass.scope.scope_id + || checkpoint.generation != pass.scope.generation + { + return Err(refused("transparent_pir_context_changed")); + } + pass.cancel.transaction(|| self.transparent_transaction(|wdb,ext| -> anyhow::Result<_> { + let mut state = load(ext)?; + let witnessed = rewind.as_ref().is_some_and(|witness| { + witness.matches(&pass.scope, &checkpoint) + && state.scope.as_ref().is_some_and(|scope| witness.matches(scope, &checkpoint)) + }); + if rewind.is_some() && !witnessed { anyhow::bail!("transparent_pir_rewind_evidence_invalid"); } + let old_records = state.records.clone(); + let missing = missing_projection(ext,&state)?; + invalidate_missing(&mut state,&missing); + let replay = state.checkpoint_id.as_ref()==Some(&checkpoint.checkpoint_id); + let repair = replay && state.projection_invalidated; + let retirement_floor = state.rewind_floor.into_iter().chain(checkpoint.rewind_to.as_ref().map(|a|a.height as u32)).min(); + if !repair { if let Some(floor)=retirement_floor { state.prepare_rewind(floor); } } + if let Some(ancestor)=&checkpoint.rewind_to { + let height=u32::try_from(ancestor.height)?; + let known=if height==0 {Some(self.network.genesis_hash().to_string())} else {wdb.get_block_hash(height.into())?.map(|h|h.to_string())}; + if known.as_deref()!=Some(&ancestor.hash) && !(known.is_none() && witnessed) {anyhow::bail!("transparent_pir_rewind_anchor_unknown");} + if !repair { state.prepare_rewind(height); } + } + let mut records = state.validate(&checkpoint)?; + let height = u32::try_from(checkpoint.anchor.height)?; + let block = wdb.block_fully_scanned()?.ok_or_else(||anyhow::anyhow!("transparent_pir_anchor_unavailable"))?; + if u32::from(block.block_height()) != height || block.block_hash().to_string() != checkpoint.anchor.hash { anyhow::bail!("transparent_pir_chain_changed"); } + if state.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id) && !state.projection_invalidated { + state.durable = true; + return Ok(state.status()); + } + let current_scripts = scripts(wdb,ext,self.network)?; + let current_id = crate::transparent_pir::digest(&serde_json::to_vec(&{let mut s=current_scripts.clone();s.sort_by(|a,b|a.script.cmp(&b.script));s.dedup();s})?); + if pass.scope.scripts.iter().any(|enrolled| !current_scripts.contains(enrolled)) { anyhow::bail!("transparent_pir_scope_changed"); } + if retirement_floor.is_some() && !checkpoint.complete { + state.records = old_records; + state.stage(&checkpoint,&[])?; + state.rewind_floor = retirement_floor; + state.durable = true; + save(ext,&state)?; + return Ok(state.status()); + } + // Snapshot confirmation authority for every transaction before ANY + // publisher metadata writes (including spend-only events). Promoting + // one output also confirms its existing pending siblings in WalletDb. + let mut original_meta=std::collections::BTreeMap::new(); + let mut pending_siblings=std::collections::BTreeSet::new(); + for (_,event) in &records { + let id=match event {TransparentEvent::Receive(e)=>e.txid.0,TransparentEvent::Spend(e)=>e.spending_txid.0}; + if let std::collections::btree_map::Entry::Vacant(entry)=original_meta.entry(id) { + let prior:Option<(Option,Option)>=ext.query_row("SELECT mined_height,tx_index FROM transactions WHERE txid=?1",[id.as_slice()],|r|Ok((r.get(0)?,r.get(1)?))).optional()?; + if prior.is_none_or(|(h,_)|h.is_none()) { + let mut stmt=ext.prepare("SELECT n.output_index FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1")?; + for index in stmt.query_map([id.as_slice()],|r|r.get::<_,u32>(0))? { + pending_siblings.insert(format!("r:{}:{}",crate::web::to_hex(&id),index?)); + if pending_siblings.len()>65_536 {anyhow::bail!("transparent_pir_state_invalid");} + } + } + entry.insert(prior); + } + } + state.unverified_outputs.get_or_insert_default().extend(pending_siblings); + if repair { + // The original source-bound checkpoint is already accepted. Rebuild + // only its erased owned effects; do not retire or rewrite independent rows. + records.retain(|(_,event)|missing.contains_key(&crate::transparent_pir::event_key(*event))); + } else if let Some(floor)=retirement_floor { retire_projection(ext,&mut state,&old_records,&records,floor)?; } + for (script,event) in &records { + let (id,position,mined) = match event { TransparentEvent::Receive(e) => (e.txid,e.transaction_index,e.height), TransparentEvent::Spend(e)=>(e.spending_txid,e.transaction_index,e.height) }; + let prior_meta=original_meta[&id.0]; + if prior_meta.is_some_and(|(h,i)|h.is_some_and(|h|h!=mined) || i.is_some_and(|i|i!=u32::from(position))) { anyhow::bail!("transparent_pir_transaction_conflict"); } + if prior_meta.is_none() { state.projected_transactions.insert(crate::web::to_hex(&id.0)); } + let meta = WalletTx::new(TxId::from_bytes(id.0),position.into(),vec![],vec![],vec![],vec![],vec![],vec![],vec![]); + let tx_ref = wdb.put_tx_meta(&meta,mined.into())?; + match event { + TransparentEvent::Receive(e) => { + let output = WalletTransparentOutput::from_parts(OutPoint::new(e.txid.0,e.output_index),TxOut::new(Zatoshis::from_u64(e.value)?, Script(zcash_script::script::Code(script.clone()))),Some(e.height.into()),None,None,None).ok_or_else(||anyhow::anyhow!("transparent_pir_script_invalid"))?; + if wdb.find_account_for_transparent_address(output.recipient_address())?.is_none() { anyhow::bail!("transparent_pir_script_not_owned"); } + let old:Option<(Vec,i64,Option)> = ext.query_row("SELECT n.script,n.value_zat,t.mined_height FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1 AND n.output_index=?2",rusqlite::params![e.txid.0.as_slice(),e.output_index],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).optional()?; + if old.is_none() {state.projected_outputs.insert(crate::transparent_pir::event_key(*event));} + if old.is_none() || prior_meta.is_none_or(|(h,_)|h.is_none()) { + state.unverified_outputs.get_or_insert_default().insert(crate::transparent_pir::event_key(*event)); + } + if old.is_some_and(|(s,v,h)| s != *script || u64::try_from(v).ok()!=Some(e.value) || h.is_some_and(|h|h!=e.height)) { anyhow::bail!("transparent_pir_output_conflict"); } + // Native upstream reconciliation retains existing spend/lock metadata. + wdb.put_transparent_output(&output,height.into(),true)?; + } + TransparentEvent::Spend(e) => { + let prior:bool=ext.query_row("SELECT EXISTS(SELECT 1 FROM transparent_spend_map s JOIN transactions t ON t.id_tx=s.spending_transaction_id WHERE t.txid=?1 AND s.prevout_txid=?2 AND s.prevout_output_index=?3) OR EXISTS(SELECT 1 FROM transparent_received_output_spends s JOIN transactions t ON t.id_tx=s.transaction_id JOIN transparent_received_outputs n ON n.id=s.transparent_received_output_id JOIN transactions p ON p.id_tx=n.transaction_id WHERE t.txid=?1 AND p.txid=?2 AND n.output_index=?3)",rusqlite::params![e.spending_txid.0.as_slice(),e.spent_txid.0.as_slice(),e.spent_output_index],|r|r.get(0))?; + if !prior {state.projected_spends.insert(crate::transparent_pir::event_key(*event));} + let found = wdb.mark_transparent_utxo_spent(&OutPoint::new(e.spent_txid.0,e.spent_output_index),tx_ref)?; + if !found && checkpoint.complete { anyhow::bail!("transparent_pir_unresolved_spend"); } + } + } + } + let unspent:u64=ext.query_row("SELECT COALESCE(SUM(n.value_zat),0) FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.mined_height IS NOT NULL AND NOT EXISTS(SELECT 1 FROM transparent_received_output_spends s WHERE s.transparent_received_output_id=n.id)",[],|r|r.get(0))?; + Zatoshis::from_u64(unspent).map_err(|_|anyhow::anyhow!("transparent_pir_amount_invalid"))?; + state.stage(&checkpoint,&records)?; + if repair && !missing_projection(ext,&state)?.is_empty() { anyhow::bail!("transparent_pir_projection_incomplete"); } + state.projection_invalidated=false; + let next_scripts = scripts(wdb,ext,self.network)?; + if next_scripts.len() != current_scripts.len() || current_id != pass.scope.scope_id { state.complete=false; } + state.durable=true; + save(ext,&state)?; + Ok(state.status()) + })) + } + + pub fn transparent_pir_status(&self) -> Result { + self.transparent_transaction(|wdb,ext| -> anyhow::Result<_> { + let mut state=load(ext)?; + let missing=missing_projection(ext,&state)?; + if invalidate_missing(&mut state,&missing) { save(ext,&state)?; } + state.durable=state.checkpoint_id.is_some(); + if let Some(scope)=&state.scope { + if wdb.block_fully_scanned()?.is_none_or(|b|u64::from(u32::from(b.block_height()))!=scope.anchor.height || b.block_hash().to_string()!=scope.anchor.hash) { state.complete=false; } + } + let mut rows=ext.prepare("SELECT t.txid,n.output_index,n.value_zat FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.mined_height IS NOT NULL AND NOT EXISTS(SELECT 1 FROM transparent_received_output_spends s WHERE s.transparent_received_output_id=n.id)")?; + let outputs=rows.query_map([],|r|Ok((r.get::<_,Vec>(0)?,r.get::<_,u32>(1)?,r.get::<_,u64>(2)?)))?; + let mut total=0u64; + for output in outputs { + let (txid,index,value)=output?; + if state.records.contains_key(&format!("r:{}:{index}",crate::web::to_hex(&txid))) { + total=total.checked_add(value).ok_or_else(||anyhow::anyhow!("transparent_pir_amount_invalid"))?; + } + } + Zatoshis::from_u64(total).map_err(|_|anyhow::anyhow!("transparent_pir_amount_invalid"))?; + let mut status=state.status();status["confirmedZat"]=serde_json::json!(total.to_string());Ok(status) + }) + } + + pub fn transparent_pir_status_for(&self, source_id: &str) -> Result { + let status = self.transparent_pir_status()?; + if status.pointer("/scope/sourceId").and_then(|v| v.as_str()) != Some(source_id) { + return Ok( + serde_json::json!({"completion":"incomplete","spendingEnabled":false,"trust":"publisher-provided-confirmed-history"}), + ); + } + Ok(status) + } + + pub fn transparent_pir_checkpoint(&self) -> Result> { + self.transparent_transaction(|_, ext| -> anyhow::Result<_> { + Ok(load(ext)? + .continuation + .as_ref() + .map(serde_json::to_string) + .transpose()?) + }) + } +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_funding.rs b/crates/z-engine/src/native/wallet/transparent_pir_funding.rs new file mode 100644 index 0000000..c28d77d --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_funding.rs @@ -0,0 +1,195 @@ +//! Funding authority is independent of PIR projection/deletion ownership. +use super::*; +use std::collections::BTreeSet; +use transparent::{address::TransparentAddress, bundle::OutPoint}; +use zcash_client_backend::{ + data_api::{ + wallet::{input_selection::ShieldingSelector, TargetHeight}, + AccountMeta, NoteFilter, ReceivedNotes, + }, + wallet::{Note, ReceivedNote, WalletTransparentOutput}, +}; + +fn key(outpoint: &OutPoint) -> String { + format!("r:{}:{}", crate::web::to_hex(outpoint.hash()), outpoint.n()) +} + +pub(super) fn check_proposal( + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, + proposal: &zcash_client_backend::proposal::Proposal, +) -> anyhow::Result<()> +where + F: zcash_primitives::transaction::fees::FeeRule, + N: Copy + Eq + Ord, +{ + let excluded = super::transparent_pir::funding_exclusions(ext)?; + if proposal + .steps() + .iter() + .flat_map(|s| s.transparent_inputs()) + .any(|o| excluded.contains(&key(o.outpoint()))) + { + anyhow::bail!("transparent_pir_funding_unverified"); + } + Ok(()) +} + +pub(super) fn propose( + db: &D, + ext: &zcash_client_sqlite::ExtensionTransaction<'_>, + network: ZNetwork, + addresses: &[TransparentAddress], + account: AccountUuid, + threshold: Zatoshis, +) -> Result> +where + D: WalletRead + + InputSource::Error>, + ::Error: std::fmt::Display, +{ + let inputs = FundingInputs { + db, + excluded: super::transparent_pir::funding_exclusions(ext).map_err(EngineError::from)?, + }; + let confirmations = shield_confirmations(network); + let (target, anchor) = db + .get_target_and_anchor_heights(confirmations.trusted()) + .map_err(|_| EngineError::SyncRequired)? + .ok_or(EngineError::SyncRequired)?; + GreedyInputSelector::new() + .propose_shielding( + &network, + &inputs, + &SingleOutputChangeStrategy::new( + StandardFeeRule::Zip317, + None, + ShieldedPool::Orchard, + DustOutputPolicy::default(), + ), + threshold, + addresses, + account, + target, + anchor, + &db.pool_migration_params(), + confirmations, + CoinbaseFilter::AllTransparentOutputs, + ) + .map_err(|e| map_funds_err(format!("propose_shielding: {e}"))) +} + +struct FundingInputs<'a, D> { + db: &'a D, + excluded: BTreeSet, +} +impl InputSource for FundingInputs<'_, D> { + type Error = D::Error; + type AccountId = D::AccountId; + type NoteRef = D::NoteRef; + fn get_spendable_note( + &self, + txid: &TxId, + protocol: ShieldedPool, + index: u32, + target: TargetHeight, + lock: LockFilter<'_>, + ) -> std::result::Result>, Self::Error> { + self.db + .get_spendable_note(txid, protocol, index, target, lock) + } + fn anchor_computable( + &self, + protocol: ShieldedPool, + height: BlockHeight, + ) -> std::result::Result { + self.db.anchor_computable(protocol, height) + } + fn select_spendable_notes( + &self, + account: Self::AccountId, + value: TargetValue, + sources: &[ShieldedPool], + target: TargetHeight, + confirmations: ConfirmationsPolicy, + exclude: &[Self::NoteRef], + lock: LockFilter<'_>, + ) -> std::result::Result, Self::Error> { + self.db.select_spendable_notes( + account, + value, + sources, + target, + confirmations, + exclude, + lock, + ) + } + fn select_unspent_notes( + &self, + account: Self::AccountId, + sources: &[ShieldedPool], + target: TargetHeight, + exclude: &[Self::NoteRef], + lock: LockFilter<'_>, + ) -> std::result::Result, Self::Error> { + self.db + .select_unspent_notes(account, sources, target, exclude, lock) + } + fn get_account_metadata( + &self, + account: Self::AccountId, + selector: &NoteFilter, + target: TargetHeight, + exclude: &[Self::NoteRef], + lock: LockFilter<'_>, + ) -> std::result::Result { + self.db + .get_account_metadata(account, selector, target, exclude, lock) + } + fn get_unspent_transparent_output( + &self, + outpoint: &OutPoint, + target: TargetHeight, + ) -> std::result::Result>, Self::Error> { + if self.excluded.contains(&key(outpoint)) { + return Ok(None); + } + self.db.get_unspent_transparent_output(outpoint, target) + } + fn get_spendable_transparent_outputs( + &self, + address: &TransparentAddress, + target: TargetHeight, + confirmations: ConfirmationsPolicy, + filter: CoinbaseFilter, + lock: LockFilter<'_>, + ) -> std::result::Result>, Self::Error> { + self.db + .get_spendable_transparent_outputs(address, target, confirmations, filter, lock) + .map(|mut outputs| { + outputs.retain(|o| !self.excluded.contains(&key(o.outpoint()))); + outputs + }) + } + fn get_spendable_transparent_outputs_for_addresses( + &self, + addresses: &[TransparentAddress], + target: TargetHeight, + confirmations: ConfirmationsPolicy, + filter: CoinbaseFilter, + lock: LockFilter<'_>, + ) -> std::result::Result>, Self::Error> { + self.db + .get_spendable_transparent_outputs_for_addresses( + addresses, + target, + confirmations, + filter, + lock, + ) + .map(|mut outputs| { + outputs.retain(|o| !self.excluded.contains(&key(o.outpoint()))); + outputs + }) + } +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_host_reorg_tests.rs b/crates/z-engine/src/native/wallet/transparent_pir_host_reorg_tests.rs new file mode 100644 index 0000000..a4d268d --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_host_reorg_tests.rs @@ -0,0 +1,1221 @@ +//! Composed engine-host recovery on a synthetic accepted chain, with genuine PIR. +use super::*; +use crate::transparent_pir::acquisition::Status; +use zcash_client_backend::data_api::OutputLockStore; +use zcash_client_backend::wallet::{LockOwner, OutputRef}; +use zcash_protocol::PoolType; + +const SOURCE: &str = "abababababababababababababababababababababababababababababababab"; +fn accepted(height: u64, fork: bool) -> String { + if height == 0 { + Network::Regtest.genesis_hash().into() + } else { + format!( + "{:02x}", + (height as u8).wrapping_add(if fork && height >= 2 { 0x80 } else { 0 }) + ) + .repeat(32) + } +} +fn walk( + scope: Scope, + previous: Option, + server: &service::InProcess, + fork: bool, + limits: Limits, + stop_below: Option, + mut apply: impl FnMut(&Checkpoint), +) -> (Checkpoint, bool) { + let mut latest = previous.clone(); + let mut pass = Pass::new(scope, previous, limits).unwrap(); + let mut rewind = false; + for _ in 0..3000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => pass.accept(request_id, answer(server, operation)).unwrap(), + Step::NeedChain { height, .. } => { + pass.accept_chain(height, accepted(height, fork)).unwrap() + } + Step::Checkpoint { checkpoint } => { + if let Some(point) = &checkpoint.rewind_to { + assert_eq!(point.hash, accepted(point.height, fork)); + rewind = true; + } + apply(&checkpoint); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + let stop = stop_below.is_some_and(|h| checkpoint.covered_through < h); + latest = Some(checkpoint); + if stop { + pass.cancel(); + return (latest.unwrap(), rewind); + } + } + Step::Stopped { status } => { + if stop_below.is_none() { + assert_eq!(status, Status::Complete); + } + return (latest.unwrap(), rewind); + } + } + } + panic!("bounded engine host fixture did not stop") +} +fn publish(path: &Path, script: &[u8], fork: bool) -> service::InProcess { + let receive = TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x11; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }); + let spend = TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid([0x22; 32]), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([0x11; 32]), + spent_output_index: 0, + }); + let events = if fork { + vec![receive] + } else { + vec![receive, spend] + }; + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + path, + &[service::ShardSpec::first( + 0, + 2, + false, + events + .into_iter() + .map(|e| (ScriptBytes::new(script.to_vec()), e)) + .collect(), + )], + service::SEAL, + |height| BlockHash::from_display_hex(&accepted(height, fork)).unwrap(), + vec![], + "regtest", + genesis, + ); + service::InProcess::serve(path) +} +#[test] +fn composed_native_same_height_fork_preserves_private_lineage_and_protected_state() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let (fixture, wallet) = runtime.block_on(async { + let f = Fixture::new(root.path()).await; + let w = f.restore(root.path()).await; + w.sync_with_wallet_source_reported_cancellable( + &f.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (f, w) + }); + use zcash_keys::encoding::AddressCodec; + let address = transparent::address::TransparentAddress::decode( + &Network::Regtest, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = script.0 .0; + let old_dir = tempfile::tempdir().unwrap(); + let old = publish(old_dir.path(), &script, false); + let independent = raw_transaction([0x91; 32], 0, 70000, &script); + wallet + .open_db() + .unwrap() + .transactionally(|db| -> anyhow::Result<()> { + decrypt_and_store_transaction(&Network::Regtest, db, &independent, Some(1.into()))?; + Ok(()) + }) + .unwrap(); + let (intent, raw) = saved_transaction(&wallet); + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + conn.execute( + "UPDATE ext_native_payments_v1 SET phase='unknown' WHERE id=?1", + [&intent], + ) + .unwrap(); + conn.execute("UPDATE transparent_received_outputs SET lock_owner=?1,lock_expiry_height=100 WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?2)",rusqlite::params![[9u8;32].as_slice(),independent.txid().as_ref()]).unwrap(); + let mut original_full = None; + for _ in 0..4 { + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + let prior = wallet + .transparent_pir_checkpoint() + .unwrap() + .map(|s| serde_json::from_str(&s).unwrap()); + let (full, _) = walk(scope, prior, &old, false, Limits::default(), None, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + original_full = Some(full); + drop(host); + if wallet.transparent_pir_status().unwrap()["completion"] == "complete-for-enrolled-scope" { + break; + } + } + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "complete-for-enrolled-scope" + ); + let original_full = original_full.unwrap(); + let native_saved = wallet.transparent_pir_checkpoint().unwrap().unwrap(); + // A canonical scanner may rewind and return to the very same accepted hash. + // The unchanged acquisition checkpoint must reconstruct its erased effects. + wallet + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + runtime + .block_on(wallet.sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + )) + .unwrap(); + // No new Pass or checkpoint may mask the unchanged-ID path. + let missing_mining: Option = conn + .query_row( + "SELECT mined_height FROM transactions WHERE txid=?1", + [[0x22u8; 32].as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + missing_mining, None, + "the real scanner must erase spending metadata" + ); + let completion_before_original_replay = + wallet.transparent_pir_status().unwrap()["completion"].clone(); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(native_saved.as_str()), + "detecting canonical loss must keep exact acquisition and debt" + ); + { + let host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + let mut no_work = Pass::new( + scope, + Some(serde_json::from_str(&native_saved).unwrap()), + Limits::default(), + ) + .unwrap(); + no_work.finish_budget().unwrap(); + assert!( + matches!( + no_work.advance().unwrap(), + Step::Stopped { + status: Status::Incomplete { .. } + } + ), + "a no-work budget stop must not replace the original checkpoint" + ); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + } + drop(wallet); + let wallet = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + let cold_completion_before_original_replay = + wallet.transparent_pir_status().unwrap()["completion"].clone(); + let mut replay = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + assert_eq!( + original_full.checkpoint_id, + serde_json::from_str::(&native_saved) + .unwrap() + .checkpoint_id + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut replay, + &serde_json::to_string(&original_full).unwrap(), + ) + .unwrap(); + drop(replay); + let restored_mining: Option = conn + .query_row( + "SELECT mined_height FROM transactions WHERE txid=?1", + [[0x22u8; 32].as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + ( + completion_before_original_replay.as_str(), + cold_completion_before_original_replay.as_str(), + restored_mining + ), + (Some("incomplete"), Some("incomplete"), Some(2)), + "status must stay incomplete until the original checkpoint ID restores spending metadata" + ); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "complete-for-enrolled-scope" + ); + assert_eq!(wallet.payment_receipt(&intent).unwrap().phase, "unknown"); + let independent_state:(u64,u32,u32)=conn.query_row("SELECT n.value_zat,t.mined_height,n.lock_expiry_height FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1",[independent.txid().as_ref()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert_eq!(independent_state, (70000, 1, 100)); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(native_saved.as_str()), + "original-ID repair must preserve exact private acquisition" + ); + let stored: Vec = conn + .query_row( + "SELECT raw FROM transactions WHERE created IS NOT NULL LIMIT 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + stored == raw, + "repair must preserve unknown transaction bytes" + ); + // Check the final transaction fence independently of the status/begin read: + // erase only the same owned mining metadata after obtaining the pass. + let mut replay = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + conn.execute( + "UPDATE transactions SET mined_height=NULL,tx_index=NULL WHERE txid=?1", + [[0x22u8; 32].as_slice()], + ) + .unwrap(); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut replay, + &serde_json::to_string(&original_full).unwrap(), + ) + .unwrap(); + conn.execute("DELETE FROM transparent_received_output_spends WHERE transaction_id IN(SELECT id_tx FROM transactions WHERE txid=?1)",[[0x22u8;32].as_slice()]).unwrap(); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut replay, + &serde_json::to_string(&original_full).unwrap(), + ) + .unwrap(); + let linked:bool=conn.query_row("SELECT EXISTS(SELECT 1 FROM transparent_received_output_spends s JOIN transactions t ON t.id_tx=s.transaction_id WHERE t.txid=?1)",[[0x22u8;32].as_slice()],|r|r.get(0)).unwrap(); + assert!( + linked, + "original-ID repair must restore the actual UTXO spend link, not just pending metadata" + ); + drop(replay); + let restored_mining: Option = conn + .query_row( + "SELECT mined_height FROM transactions WHERE txid=?1", + [[0x22u8; 32].as_slice()], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + restored_mining, + Some(2), + "transaction guard must repair without a preceding status read" + ); + // Canonical SQLite rollback and independently selected compact rescan own chain authority. + wallet + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + fixture.mode.store(4, Ordering::Release); + runtime + .block_on(wallet.sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + )) + .unwrap(); + assert_eq!( + wallet + .open_db() + .unwrap() + .get_block_hash(2.into()) + .unwrap() + .unwrap() + .to_string(), + accepted(2, true) + ); + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(native_saved.as_str()), + "native begin must retain exact private evidence" + ); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + let new_dir = tempfile::tempdir().unwrap(); + let new = publish(new_dir.path(), &script, true); + let prior = Some(serde_json::from_str(&native_saved).unwrap()); + let canonical_before_partial = wallet.transparent_pir_status().unwrap()["confirmedZat"].clone(); + let (_, rewind) = walk(scope, prior, &new, true, Limits::default(), None, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + if !cp.complete { + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + canonical_before_partial, + "header-only progress must preserve canonical effects until verified rewind" + ); + assert_eq!(wallet.payment_receipt(&intent).unwrap().phase, "unknown"); + } + }); + assert!( + rewind, + "core must emit independently authenticated rollback" + ); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!(wallet.payment_receipt(&intent).unwrap().phase, "unknown"); + let stored: Vec = conn + .query_row( + "SELECT raw FROM transactions WHERE created IS NOT NULL LIMIT 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(stored == raw, "unknown transaction bytes must be unchanged"); + let independent_state:(u64,u32,u32)=conn.query_row("SELECT n.value_zat,t.mined_height,n.lock_expiry_height FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1",[independent.txid().as_ref()],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap(); + assert_eq!(independent_state, (70000, 1, 100)); + drop(host); + drop(wallet); + let reopened = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!(reopened.payment_receipt(&intent).unwrap().phase, "unknown"); + + assert!(!fixture + .calls + .lock() + .unwrap() + .iter() + .any(|m| matches!(m.as_str(), "GetAddressUtxos" | "SendTransaction"))); +} + +#[test] +fn composed_web_same_height_fork_preserves_private_lineage_and_protected_state() { + let mut derived = crate::web::WebWallet::from_account( + account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(), + 1, + ) + .unwrap(); + let script = derived.public_transparent_scripts().unwrap()[0].0.clone(); + let independent = raw_transaction([0x91; 32], 0, 70000, &script); + let pending = raw_transaction([0x92; 32], 0, 1000, &script); + let mut raw = vec![]; + pending.write(&mut raw).unwrap(); + let old_dir = tempfile::tempdir().unwrap(); + let old = publish(old_dir.path(), &script, false); + let new_dir = tempfile::tempdir().unwrap(); + let new = publish(new_dir.path(), &script, true); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account, 1).unwrap(); + let mut blocks: Vec<_> = (1..=2) + .map(|h| block(h, Network::Regtest.genesis_hash())) + .collect(); + blocks[0].vtx.push(CompactTx { + index: 0, + txid: vec![0x61; 32], + ..Default::default() + }); + blocks[1].vtx.push(CompactTx { + index: 0, + txid: vec![0x62; 32], + ..Default::default() + }); + blocks[0].vtx.push(CompactTx { + index: 1, + txid: independent.txid().as_ref().to_vec(), + vout: vec![TxOut { + value: 70000, + script_pub_key: script.clone(), + }], + ..Default::default() + }); + let compact = crate::web::encode_delimited(blocks); + web.apply_compact_blocks_blob(&compact).unwrap(); + web.apply_transparent_blocks(&compact).unwrap(); + for _ in 0..4 { + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + let prior = web + .transparent_pir_checkpoint() + .map(|s| serde_json::from_str(&s).unwrap()); + walk(scope, prior, &old, false, Limits::default(), None, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + if web.transparent_pir_status()["completion"] == "complete-for-enrolled-scope" { + break; + } + } + assert_eq!( + web.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + let web_saved = web.transparent_pir_checkpoint().unwrap(); + web.store_pending_tx(&Transaction::read(raw.as_slice(), BranchId::Nu6_3).unwrap()) + .unwrap(); + let pending = web.pending_raw_txs(); + web.lock_outputs( + &[OutputRef::new(independent.txid(), PoolType::TRANSPARENT, 0)], + LockOwner::new([9; 32]), + 100.into(), + ) + .unwrap(); + web.rewind_to_height_checked(1).unwrap(); + assert!( + web.transparent_pir_checkpoint().as_deref() == Some(web_saved.as_str()), + "real web rewind must retain private evidence" + ); + assert!( + web.begin_transparent_pir(SOURCE).is_err(), + "lower target is closed until canonical scanning catches up" + ); + assert!( + web.pending_raw_txs() == pending, + "pending transaction bytes must be unchanged" + ); + let mut second = block(2, Network::Regtest.genesis_hash()); + second.hash = vec![0x82; 32]; + web.apply_compact_blocks_blob(&crate::web::encode_delimited([second])) + .unwrap(); + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + assert!( + web.transparent_pir_checkpoint().as_deref() == Some(web_saved.as_str()), + "same-height web begin must retain evidence" + ); + let (_, rewind) = walk( + scope, + Some(serde_json::from_str(&web_saved).unwrap()), + &new, + true, + Limits::default(), + None, + |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }, + ); + assert!(rewind); + assert_eq!(web.transparent_pir_status()["confirmedZat"], "150000"); + assert!( + web.pending_raw_txs() == pending, + "pending transaction bytes must be unchanged" + ); + assert_eq!(web.output_locks.len(), 1); + assert!(web.utxos.iter().any( + |u| u.txid == crate::web::to_hex(independent.txid().as_ref()) + && u.value_zat == 70000 + && u.height == 1 + )); + let web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + assert_eq!(web.transparent_pir_status()["confirmedZat"], "150000"); + assert!( + web.pending_raw_txs() == pending, + "pending transaction bytes must be unchanged" + ); +} + +#[test] +fn composed_native_web_scope_growth_accepts_lower_core_frontier_and_cold_recovers() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let (fixture, wallet) = runtime.block_on(async { + let f = Fixture::new(root.path()).await; + let w = f.restore(root.path()).await; + w.sync_with_wallet_source_reported_cancellable( + &f.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (f, w) + }); + let host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let native_initial: Scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + drop(host); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account.clone(), 1).unwrap(); + web.apply_compact_blocks_blob(&crate::web::encode_delimited( + (1..=2).map(|h| block(h, Network::Regtest.genesis_hash())), + )) + .unwrap(); + let web_initial: Scope = + serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + let mut future = crate::web::WebWallet::from_account(account, 1).unwrap(); + future.next_diversifier = 128; + let future_script = future + .public_transparent_scripts() + .unwrap() + .into_iter() + .map(|(s, _)| s) + .find(|s| { + let hex = crate::web::to_hex(s); + !native_initial.scripts.iter().any(|s| s.script == hex) + && !web_initial.scripts.iter().any(|s| s.script == hex) + }) + .unwrap(); + let native_future_key = + UnifiedSpendingKey::from_seed(&ZNetwork::Regtest, &[7; 32], Zip32AccountId::ZERO) + .unwrap() + .to_unified_full_viewing_key(); + let public = native_future_key + .transparent() + .unwrap() + .derive_address_pubkey( + transparent::keys::TransparentKeyScope::EXTERNAL, + transparent::keys::NonHardenedChildIndex::from_index(0).unwrap(), + ) + .unwrap(); + let address = transparent::address::TransparentAddress::from_pubkey(&public); + let native_future_script: transparent::address::Script = address.script().into(); + let publication = tempfile::tempdir().unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + publication.path(), + &[service::ShardSpec::first( + 0, + 2, + false, + vec![ + ( + ScriptBytes::new(future_script.clone()), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x55; 32]), + transaction_index: 0, + output_index: 0, + value: 17000, + coinbase: false, + }), + ), + ( + ScriptBytes::new(native_future_script.0 .0), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x66; 32]), + transaction_index: 1, + output_index: 0, + value: 17000, + coinbase: false, + }), + ), + ], + )], + service::SEAL, + |h| BlockHash::from_display_hex(&accepted(h, false)).unwrap(), + vec![], + "regtest", + genesis, + ); + let server = service::InProcess::serve(publication.path()); + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + walk( + serde_json::from_str(&host.scope_json().unwrap()).unwrap(), + None, + &server, + false, + Limits::default(), + None, + |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }, + ); + drop(host); + let native_saved = wallet.transparent_pir_checkpoint().unwrap().unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["coveredThrough"], + 2 + ); + walk( + web_initial, + None, + &server, + false, + Limits::default(), + None, + |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }, + ); + let web_saved = web.transparent_pir_checkpoint().unwrap(); + assert_eq!(web.transparent_pir_status()["coveredThrough"], 2); + // A real account import expands the wallet-owned inventory. Both newly + // imported and issued receivers have independent publisher fixture events. + { + let mut db = wallet.open_db().unwrap(); + let ufvk = + UnifiedSpendingKey::from_seed(&ZNetwork::Regtest, &[7; 32], Zip32AccountId::ZERO) + .unwrap() + .to_unified_full_viewing_key(); + let birthday = AccountBirthday::from_parts( + zcash_client_backend::data_api::chain::ChainState::empty( + 2.into(), + zcash_primitives::block::BlockHash([2; 32]), + ), + None, + ); + db.import_account_ufvk( + "scope growth fixture", + &ufvk, + &birthday, + AccountPurpose::Spending { derivation: None }, + None, + ) + .unwrap(); + } + let pass = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let new_scope: Scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + drop(pass); + assert_ne!(new_scope.scope_id, native_initial.scope_id); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(native_saved.as_str()), + "scope growth must retain acquisition" + ); + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let (partial, _) = walk( + new_scope, + Some(serde_json::from_str(&native_saved).unwrap()), + &server, + false, + Limits::default(), + Some(2), + |cp| { + assert!(cp.rewind_to.is_none()); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }, + ); + drop(host); + assert!(!partial.complete); + assert_eq!(partial.covered_through, 0); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + drop(wallet); + let wallet = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + for _ in 0..4 { + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + let cp = wallet + .transparent_pir_checkpoint() + .unwrap() + .map(|s| serde_json::from_str(&s).unwrap()); + walk(scope, cp, &server, false, Limits::default(), None, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + drop(host); + if wallet.transparent_pir_status().unwrap()["completion"] == "complete-for-enrolled-scope" { + break; + } + } + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "17000" + ); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "complete-for-enrolled-scope" + ); + web.next_diversifier = 128; + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + assert!( + web.transparent_pir_checkpoint().as_deref() == Some(web_saved.as_str()), + "web growth must retain acquisition" + ); + let (partial, _) = walk( + scope, + Some(serde_json::from_str(&web_saved).unwrap()), + &server, + false, + Limits::default(), + Some(2), + |cp| { + assert!(cp.rewind_to.is_none()); + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }, + ); + assert_eq!(partial.covered_through, 0); + assert!(!partial.complete); + let mut web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + for _ in 0..4 { + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + let cp = web + .transparent_pir_checkpoint() + .map(|s| serde_json::from_str(&s).unwrap()); + walk(scope, cp, &server, false, Limits::default(), None, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + if web.transparent_pir_status()["completion"] == "complete-for-enrolled-scope" { + break; + } + } + assert_eq!(web.transparent_pir_status()["confirmedZat"], "17000"); + assert_eq!( + web.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); +} + +fn seed_debt( + scope: Scope, + server: &service::InProcess, + mut apply: impl FnMut(&Checkpoint), +) -> Checkpoint { + let mut pass = Pass::new(scope, None, Limits::default()).unwrap(); + let mut pages = 0; + for _ in 0..1000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + pages += usize::from(matches!( + operation, + Operation::Query { + table: Table::Pages { .. }, + .. + } + )); + pass.accept(request_id, answer(server, operation)).unwrap(); + } + Step::NeedChain { height, .. } => { + pass.accept_chain(height, accepted(height, false)).unwrap() + } + Step::Checkpoint { checkpoint } => { + apply(&checkpoint); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + if pages == 1 { + assert!(checkpoint.pending_pages > 0); + assert!(!checkpoint.complete); + pass.cancel(); + return checkpoint; + } + } + Step::Stopped { .. } => panic!("malformed head must checkpoint before stopping"), + } + } + panic!("bounded debt fixture did not stop") +} +fn undecided_budget(scope: Scope, prior: Checkpoint) { + let mut pass = Pass::new(scope, Some(prior), Limits::default()).unwrap(); + assert!(matches!( + pass.advance().unwrap(), + Step::Exchange { + operation: Operation::Init, + .. + } + )); + pass.finish_budget().unwrap(); + assert!( + matches!( + pass.advance().unwrap(), + Step::Stopped { + status: Status::Incomplete { .. } + } + ), + "undecided lineage must not replace the private journal" + ); +} +fn pay_debt( + scope: Scope, + prior: Checkpoint, + server: &service::InProcess, + mut apply: impl FnMut(&Checkpoint), +) -> (usize, String) { + let mut pass = Pass::new( + scope, + Some(prior), + Limits { + max_queries: 1, + ..Limits::default() + }, + ) + .unwrap(); + let mut pages = 0; + for _ in 0..1000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + pages += usize::from(matches!( + operation, + Operation::Query { + table: Table::Pages { .. }, + .. + } + )); + pass.accept(request_id, answer(server, operation)).unwrap(); + } + Step::NeedChain { height, .. } => { + pass.accept_chain(height, accepted(height, false)).unwrap() + } + Step::Checkpoint { checkpoint } => { + apply(&checkpoint); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + } + Step::Stopped { + status: Status::Incomplete { reason }, + } => return (pages, reason), + Step::Stopped { .. } => panic!("malformed history cannot be complete"), + } + } + panic!("bounded cover payment did not stop") +} +#[test] +fn composed_native_web_fork_and_cold_budget_keep_real_owed_cover_queries() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let (fixture, wallet) = runtime.block_on(async { + let f = Fixture::new(root.path()).await; + let w = f.restore(root.path()).await; + w.sync_with_wallet_source_reported_cancellable( + &f.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (f, w) + }); + use zcash_keys::encoding::AddressCodec; + let address = transparent::address::TransparentAddress::decode( + &Network::Regtest, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = script.0 .0; + let events = (0..3) + .map(|i| { + ( + ScriptBytes::new(script.clone()), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: if i == 0 { 1 } else { 2 }, + txid: Txid([0x70 + i; 32]), + transaction_index: u16::from(i), + output_index: 0, + value: 100, + coinbase: false, + }), + ) + }) + .collect(); + let mut shard = service::ShardSpec::first(0, 2, false, events); + shard.malformed_page_heads = true; + let publication = tempfile::tempdir().unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + publication.path(), + &[shard], + service::SEAL, + |h| BlockHash::from_display_hex(&accepted(h, false)).unwrap(), + vec![], + "regtest", + genesis, + ); + let server = service::InProcess::serve(publication.path()); + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + seed_debt(scope, &server, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + drop(host); + let saved = wallet.transparent_pir_checkpoint().unwrap().unwrap(); + wallet + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + assert!(wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()) + ) + .is_err()); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(saved.as_str()), + "lower native target must not discard owed queries" + ); + fixture.mode.store(4, Ordering::Release); + runtime + .block_on(wallet.sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + )) + .unwrap(); + let host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + undecided_budget( + serde_json::from_str(&host.scope_json().unwrap()).unwrap(), + serde_json::from_str(&saved).unwrap(), + ); + drop(host); + assert!( + wallet.transparent_pir_checkpoint().unwrap().as_deref() == Some(saved.as_str()), + "native fork budget must retain debt exactly" + ); + drop(wallet); + let wallet = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + wallet + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + fixture.mode.store(0, Ordering::Release); + runtime + .block_on(wallet.sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + )) + .unwrap(); + let mut native_counts = vec![]; + let mut native_reasons = vec![]; + for _ in 0..3 { + let mut host = wallet + .begin_transparent_pir_with_source( + &fixture.source, + SOURCE, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&host.scope_json().unwrap()).unwrap(); + let cp = + serde_json::from_str(&wallet.transparent_pir_checkpoint().unwrap().unwrap()).unwrap(); + let (count, reason) = pay_debt(scope, cp, &server, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut host, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + drop(host); + native_counts.push(count); + native_reasons.push(reason); + } + assert_eq!(native_counts, vec![1, 1, 1]); + assert_eq!( + native_reasons, + vec!["query_budget", "query_budget", "content_failed"] + ); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account, 1).unwrap(); + web.apply_compact_blocks_blob(&crate::web::encode_delimited( + (1..=2).map(|h| block(h, Network::Regtest.genesis_hash())), + )) + .unwrap(); + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + seed_debt(scope, &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + let saved = web.transparent_pir_checkpoint().unwrap(); + web.rewind_to_height_checked(1).unwrap(); + assert!( + web.transparent_pir_checkpoint().as_deref() == Some(saved.as_str()), + "canonical web rewind must retain genuine owed queries" + ); + assert!(web.begin_transparent_pir(SOURCE).is_err()); + let mut web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + let mut second = block(2, Network::Regtest.genesis_hash()); + second.hash = vec![0x82; 32]; + web.apply_compact_blocks_blob(&crate::web::encode_delimited([second])) + .unwrap(); + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + undecided_budget(scope, serde_json::from_str(&saved).unwrap()); + assert!( + web.transparent_pir_checkpoint().as_deref() == Some(saved.as_str()), + "web fork budget must retain debt exactly" + ); + let mut web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + web.rewind_to_height_checked(1).unwrap(); + web.apply_compact_blocks_blob(&crate::web::encode_delimited([block( + 2, + Network::Regtest.genesis_hash(), + )])) + .unwrap(); + let mut web_counts = vec![]; + let mut web_reasons = vec![]; + for _ in 0..3 { + let scope = serde_json::from_str(&web.begin_transparent_pir(SOURCE).unwrap()).unwrap(); + let cp = serde_json::from_str(&web.transparent_pir_checkpoint().unwrap()).unwrap(); + let (count, reason) = pay_debt(scope, cp, &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + web_counts.push(count); + web_reasons.push(reason); + web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + } + assert_eq!(web_counts, vec![1, 1, 1]); + assert_eq!( + web_reasons, + vec!["query_budget", "query_budget", "content_failed"] + ); + assert_eq!(web.transparent_pir_status()["completion"], "incomplete"); +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs b/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs new file mode 100644 index 0000000..d00d532 --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_integration_tests.rs @@ -0,0 +1,1237 @@ +//! Real v12 PIR retrieval projected into both canonical wallets, checked against +//! independent serialized transaction ingestion. This fixture tests wallet data +//! semantics; it does not claim consensus-valid signatures or proof of work. +use super::*; +use crate::keys::account_from_mnemonic; +use crate::transparent_pir::acquisition::{ + Checkpoint, Limits, Operation, Pass, Reply, Scope, Step, Table, +}; +use crate::Network; +use base64::{engine::general_purpose::STANDARD, Engine}; +use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; +use transparent_filter::{BlockHash, ScriptBytes}; +use zcash_client_backend::proto::compact_formats::{CompactTx, CompactTxIn, TxOut}; + +#[allow(dead_code)] +#[path = "../../../../z-transparent-pir-v2/tests/fixture.rs"] +mod service; + +fn answer(server: &service::InProcess, operation: Operation) -> Reply { + let table = |t: Table| match t { + Table::Directory => "directory".into(), + Table::Pages { child } => format!("pages-{child}"), + }; + let (method, path, body) = match operation { + Operation::Init => ("GET", "/v2/shards/init".into(), vec![]), + Operation::Map => ("GET", "/v2/filters/shards".into(), vec![]), + Operation::Filter { shard_id } => ( + "GET", + format!("/v2/filters/shards/{shard_id}/filter"), + vec![], + ), + Operation::Manifest { shard_id, revision } => ( + "GET", + format!("/v2/shards/{shard_id}/revisions/{revision}/manifest"), + vec![], + ), + Operation::Setup { + shard_id, + revision, + table: t, + segment, + } => ( + "GET", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/setup/{}/{segment}", + table(t) + ), + vec![], + ), + Operation::Query { + shard_id, + revision, + table: t, + body, + } => ( + "POST", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/query/{}", + table(t) + ), + STANDARD.decode(body).unwrap(), + ), + }; + let response = server.answer( + axum::http::Request::builder() + .method(method) + .uri(path) + .body(body) + .unwrap(), + ); + assert_eq!( + response.status(), + 200, + "{}", + String::from_utf8_lossy(response.body()) + ); + Reply::Data { + body: STANDARD.encode(response.body()), + } +} +fn drive( + scope: Scope, + checkpoint: Option, + server: &service::InProcess, + mut apply: impl FnMut(&Checkpoint), +) -> Checkpoint { + let mut pass = Pass::new( + scope, + checkpoint, + Limits { + max_exchanges: 256, + ..Limits::default() + }, + ) + .unwrap(); + let mut latest = None; + for _ in 0..2000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => pass.accept(request_id, answer(server, operation)).unwrap(), + Step::NeedChain { height, hash } => { + let expected = if height == 0 { + Network::Regtest.genesis_hash().into() + } else { + format!("{:02x}", height as u8).repeat(32) + }; + assert_eq!(hash, expected); + pass.accept_chain(height, expected).unwrap(); + } + Step::Checkpoint { checkpoint } => { + apply(&checkpoint); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + latest = Some(checkpoint); + } + Step::Stopped { .. } => return latest.unwrap(), + } + } + panic!("bounded fixture did not finish") +} +fn raw_transaction(previous: [u8; 32], index: u32, value: u64, script: &[u8]) -> Transaction { + let mut raw = vec![]; + raw.extend(1u32.to_le_bytes()); + raw.push(1); + raw.extend(previous); + raw.extend(index.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + raw.push(1); + raw.extend(value.to_le_bytes()); + raw.push(u8::try_from(script.len()).unwrap()); + raw.extend(script); + raw.extend(0u32.to_le_bytes()); + let tx = Transaction::read(raw.as_slice(), BranchId::Sprout).unwrap(); + let mut canonical = vec![]; + tx.write(&mut canonical).unwrap(); + assert_eq!(canonical, raw); + tx +} +fn rows(wallet: &NativeWallet) -> Vec<(Vec, u32, Vec, u64, u32, Option>)> { + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let mut stmt=conn.prepare("SELECT t.txid,n.output_index,n.script,n.value_zat,t.mined_height,s.txid FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id LEFT JOIN transparent_received_output_spends l ON l.transparent_received_output_id=n.id LEFT JOIN transactions s ON s.id_tx=l.transaction_id ORDER BY t.txid,n.output_index").unwrap(); + stmt.query_map([], |r| { + Ok(( + r.get(0)?, + r.get(1)?, + r.get(2)?, + r.get(3)?, + r.get(4)?, + r.get(5)?, + )) + }) + .unwrap() + .collect::>>() + .unwrap() +} + +#[test] +fn transparent_pir_real_service_matches_independent_transaction_native_and_web_ingestion() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let reference_root = tempfile::tempdir().unwrap(); + let (fixture, wallet, reference_fixture, reference) = runtime.block_on(async { + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let reference_fixture = Fixture::new(reference_root.path()).await; + let reference = reference_fixture.restore(reference_root.path()).await; + reference + .sync_with_wallet_source_reported_cancellable( + &reference_fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (fixture, wallet, reference_fixture, reference) + }); + use zcash_keys::encoding::AddressCodec; + let address = transparent::address::TransparentAddress::decode( + &Network::Regtest, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = script.0 .0; + let funding = raw_transaction([9; 32], 0, 150000, &script); + let outsider: transparent::address::Script = + transparent::address::TransparentAddress::PublicKeyHash([7; 20]) + .script() + .into(); + let spending = raw_transaction(*funding.txid().as_ref(), 0, 140000, &outsider.0 .0); + // The source publication is derived from parsed serialized transactions. + let received = funding.transparent_bundle().unwrap().vout[0].clone(); + let consumed = spending.transparent_bundle().unwrap().vin[0].prevout(); + let receive = TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid(*funding.txid().as_ref()), + transaction_index: 1, + output_index: 0, + value: u64::from(received.value()), + coinbase: funding.transparent_bundle().unwrap().is_coinbase(), + }); + let spend = TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid(*spending.txid().as_ref()), + transaction_index: 1, + input_index: 0, + spent_txid: Txid(*consumed.hash()), + spent_output_index: consumed.n(), + }); + let publication = tempfile::tempdir().unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + publication.path(), + &[service::ShardSpec::first( + 0, + 2, + false, + vec![ + (ScriptBytes::new(script.clone()), receive), + (ScriptBytes::new(script.clone()), spend), + ], + )], + service::SEAL, + |h| { + if h == 0 { + genesis + } else { + BlockHash::from_internal_bytes([h as u8; 32]) + } + }, + vec![], + "regtest", + genesis, + ); + let server = service::InProcess::serve(publication.path()); + // Independent canonical reference consumes whole transactions, not source events. + let mut db = reference.open_db().unwrap(); + db.transactionally(|wdb| -> anyhow::Result<()> { + decrypt_and_store_transaction(&Network::Regtest, wdb, &funding, Some(1.into()))?; + decrypt_and_store_transaction(&Network::Regtest, wdb, &spending, Some(2.into()))?; + Ok(()) + }) + .unwrap(); + drop(db); + let mut native = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: Scope = serde_json::from_str(&native.scope_json().unwrap()).unwrap(); + let checkpoint = drive(scope, None, &server, |cp| { + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &serde_json::to_string(cp).unwrap(), + ) + .unwrap(); + }); + assert!(checkpoint.complete); + assert_eq!(rows(&wallet), rows(&reference)); + assert_eq!(rows(&wallet).len(), 1); + assert_eq!(rows(&wallet)[0].3, 150000); + assert_eq!(rows(&wallet)[0].5, Some(spending.txid().as_ref().to_vec())); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!(wallet.history(20).unwrap().len(), 2); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account.clone(), 1).unwrap(); + let mut web_reference = crate::web::WebWallet::from_account(account, 1).unwrap(); + let mut first = block(1, Network::Regtest.genesis_hash()); + let mut second = block(2, Network::Regtest.genesis_hash()); + for b in [&mut first, &mut second] { + b.vtx.push(CompactTx { + index: 0, + txid: vec![0x66 + b.height as u8; 32], + ..Default::default() + }); + } + first.vtx.push(CompactTx { + index: 1, + txid: funding.txid().as_ref().to_vec(), + vout: vec![TxOut { + value: 150000, + script_pub_key: script.clone(), + }], + ..Default::default() + }); + second.vtx.push(CompactTx { + index: 1, + txid: spending.txid().as_ref().to_vec(), + vin: vec![CompactTxIn { + prevout_txid: funding.txid().as_ref().to_vec(), + prevout_index: 0, + }], + vout: vec![TxOut { + value: 140000, + script_pub_key: outsider.0 .0, + }], + ..Default::default() + }); + let compact = crate::web::encode_delimited(vec![first, second]); + web.apply_compact_blocks_blob(&compact).unwrap(); + web_reference.apply_compact_blocks_blob(&compact).unwrap(); + web_reference.apply_transparent_blocks(&compact).unwrap(); + let scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + drive(scope, None, &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + let saved = web.to_snapshot().unwrap(); + assert!(!saved.is_empty()); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + let actual = web.history(20); + let expected = web_reference.history(20); + assert_eq!(actual.len(), 2); + assert_eq!(actual.len(), expected.len()); + for e in &actual { + let r = expected.iter().find(|r| r.txid == e.txid).unwrap(); + assert_eq!( + (e.transparent_received, e.transparent_spent), + (r.transparent_received, r.transparent_spent) + ); + } + assert_eq!(web.transparent_pir_status()["confirmedZat"], "0"); + let restored = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.history(20).len(), 2); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "0"); + // Finish any gap expansion, then reset and rescan to the unchanged tip. + let scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let prior = serde_json::from_str(&web.transparent_pir_checkpoint().unwrap()).unwrap(); + drive(scope, Some(prior), &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + assert_eq!( + web.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + let continuation = web.transparent_pir_checkpoint().unwrap(); + web.reset_scan(); + assert!(web.history(20).is_empty()); + assert!(web.transparent_pir_checkpoint().unwrap() == continuation); + let mut web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + web.apply_compact_blocks_blob(&compact).unwrap(); + let scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let prior = serde_json::from_str(&web.transparent_pir_checkpoint().unwrap()).unwrap(); + drive(scope, Some(prior), &server, |cp| { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(cp).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + }); + assert_eq!( + web.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + assert_eq!(web.transparent_pir_status()["confirmedZat"], "0"); + assert_eq!(web.history(20).len(), 2); + assert_eq!( + web.balance().transparent_available + web.balance().transparent_pending, + 0 + ); + drop(native); + drop(wallet); + let reopened = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!(rows(&reopened), rows(&reference)); + assert!(!fixture.calls.lock().unwrap().iter().any(|m| matches!( + m.as_str(), + "GetTransaction" | "GetAddressUtxos" | "SendTransaction" + ))); + drop(server); + drop(fixture); + drop(reference_fixture); + drop(runtime); +} + +#[test] +fn web_reset_scan_keeps_real_malformed_page_cover_debt_across_cold_resumes() { + use crate::transparent_pir::acquisition::Status; + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut web = crate::web::WebWallet::from_account(account, 1).unwrap(); + web.next_diversifier = 1; + let compact = + crate::web::encode_delimited((1..=3).map(|h| block(h, Network::Regtest.genesis_hash()))); + web.apply_compact_blocks_blob(&compact).unwrap(); + let script = web.public_transparent_scripts().unwrap()[0].0.clone(); + let events = (1..=3) + .map(|h| { + ( + ScriptBytes::new(script.clone()), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: h, + txid: Txid([h as u8 + 0x70; 32]), + transaction_index: 0, + output_index: 0, + value: 100, + coinbase: false, + }), + ) + }) + .collect(); + let mut shard = service::ShardSpec::first(0, 3, true, events); + shard.malformed_page_heads = true; + let publication = tempfile::tempdir().unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + service::publish_for_network( + publication.path(), + &[shard], + service::SEAL, + |h| { + if h == 0 { + genesis + } else { + BlockHash::from_internal_bytes([h as u8; 32]) + } + }, + vec![], + "regtest", + genesis, + ); + let server = service::InProcess::serve(publication.path()); + let scope: Scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let mut pass = Pass::new(scope, None, Limits::default()).unwrap(); + let mut first_pages = 0; + let mut cancelled = false; + for _ in 0..200 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + first_pages += usize::from(matches!( + operation, + Operation::Query { + table: Table::Pages { .. }, + .. + } + )); + pass.accept(request_id, answer(&server, operation)).unwrap(); + } + Step::NeedChain { height, hash } => pass.accept_chain(height, hash).unwrap(), + Step::Checkpoint { checkpoint } => { + web.apply_transparent_pir_checkpoint(&serde_json::to_string(&checkpoint).unwrap()) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&checkpoint.checkpoint_id) + .unwrap(); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + if first_pages == 1 { + pass.cancel(); + cancelled = true; + break; + } + } + Step::Stopped { .. } => { + panic!("must stop immediately after the failed head checkpoint") + } + } + } + assert!(cancelled); + drop(pass); + let continuation = web.transparent_pir_checkpoint().unwrap(); + web.reset_scan(); + assert!(web.transparent_pir_checkpoint().unwrap() == continuation); + let mut web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + web.apply_compact_blocks_blob(&compact).unwrap(); + let mut pages = Vec::new(); + let mut reasons = Vec::new(); + for _ in 0..3 { + let scope = + serde_json::from_str(&web.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let checkpoint = serde_json::from_str(&web.transparent_pir_checkpoint().unwrap()).unwrap(); + let mut pass = Pass::new( + scope, + Some(checkpoint), + Limits { + max_queries: 1, + ..Limits::default() + }, + ) + .unwrap(); + let mut count = 0; + let mut stopped = false; + for _ in 0..200 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + count += usize::from(matches!( + operation, + Operation::Query { + table: Table::Pages { .. }, + .. + } + )); + pass.accept(request_id, answer(&server, operation)).unwrap(); + } + Step::NeedChain { height, hash } => pass.accept_chain(height, hash).unwrap(), + Step::Checkpoint { checkpoint } => { + web.apply_transparent_pir_checkpoint( + &serde_json::to_string(&checkpoint).unwrap(), + ) + .unwrap(); + web.acknowledge_transparent_pir_checkpoint(&checkpoint.checkpoint_id) + .unwrap(); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + } + Step::Stopped { + status: Status::Incomplete { reason }, + } => { + reasons.push(reason); + stopped = true; + break; + } + Step::Stopped { .. } => panic!("malformed content must remain incomplete"), + } + } + assert!(stopped); + pages.push(count); + web = crate::web::WebWallet::from_snapshot(&web.to_snapshot().unwrap()).unwrap(); + } + assert_eq!(pages, vec![1, 1, 1]); + assert_eq!( + reasons, + vec!["query_budget", "query_budget", "content_failed"] + ); + assert_eq!(web.transparent_pir_status()["confirmedZat"], "200"); + assert_eq!(web.history(20).len(), 2); + assert_eq!(web.transparent_pir_status()["completion"], "incomplete"); +} + +#[path = "transparent_pir_host_reorg_tests.rs"] +mod host_reorg; + +#[test] +fn historical_birthday_gap_provisional_replacement_accepts_independent_ancestor() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let (mut fixture, mut wallet) = runtime.block_on(async { + let fixture = Fixture::new_with_tip(root.path(), 4).await; + let wallet = NativeWallet::restore_with_wallet_source( + root.path(), + PUBLIC_SEED, + &fixture.source, + 3, + auth(), + 0, + ) + .await + .unwrap() + .0; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + (fixture, wallet) + }); + let latest = runtime.block_on(async { + let mut client = zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient::new(fixture.channel.clone()); + client.get_latest_block(zcash_client_backend::proto::service::ChainSpec::default()).await.unwrap().into_inner() + }); + assert_eq!(latest.height, 4); + assert_eq!(latest.hash, vec![4; 32]); + let birthday = 3u64; + let missing = wallet + .open_db() + .unwrap() + .get_block_hash(1.into()) + .unwrap() + .is_none(); + assert!( + missing, + "historical ancestor must be outside stored canonical block rows" + ); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + let hash = |height| { + if height == 0 { + genesis + } else { + BlockHash::from_display_hex(&format!("{:02x}", height as u8).repeat(32)).unwrap() + } + }; + let old_dir = tempfile::tempdir().unwrap(); + let old_map = service::publish_for_network( + old_dir.path(), + &[ + service::ShardSpec::first(0, 1, true, vec![]), + service::ShardSpec::first(2, 4, false, vec![]), + ], + service::SEAL, + hash, + vec![], + "regtest", + genesis, + ); + let old_server = service::InProcess::serve(old_dir.path()); + let mut previous = None; + let mut observed = std::collections::BTreeMap::new(); + let mut old_point_reads = 0; + let mut prior_tail = None; + for phase in 0..4 { + if phase == 3 { + fixture = runtime.block_on(Fixture::new_with_tip(root.path(), 5)); + runtime + .block_on(wallet.sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + )) + .unwrap(); + } + wallet = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + let next_dir = tempfile::tempdir().unwrap(); + let target = if phase == 3 { 5 } else { 4 }; + let mut tail = service::ShardSpec::first(2, target, false, vec![]); + tail.revision = if phase == 3 { 2 } else { 1 }; + tail.supersedes = if phase == 3 { + prior_tail.clone().unwrap() + } else { + old_map.shards[1].manifest_digest.clone() + }; + let next_server = if phase > 0 { + let map = service::publish_for_network( + next_dir.path(), + &[service::ShardSpec::first(0, 1, true, vec![]), tail], + service::SEAL, + hash, + vec![], + "regtest", + genesis, + ); + prior_tail = Some(map.shards[1].manifest_digest.clone()); + Some(service::InProcess::serve(next_dir.path())) + } else { + None + }; + let server = next_server.as_ref().unwrap_or(&old_server); + let mut native = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: Scope = serde_json::from_str(&native.scope_json().unwrap()).unwrap(); + assert_eq!(scope.anchor.height, target); + if phase == 1 { + // The continuation's SHA is a content address, not authentication. + // Recompute it after tampering, while leaving the owner's saved + // checkpoint, rows and sidecar untouched. Owned lineage must refuse. + let mut forged: Checkpoint = previous.clone().unwrap(); + let mut bytes = STANDARD.decode(&forged.continuation).unwrap(); + let needle = "01".repeat(32).into_bytes(); + let offset = bytes + .windows(64) + .rposition(|window| window == needle) + .unwrap(); + bytes[offset..offset + 64].copy_from_slice("ee".repeat(32).as_bytes()); + forged.continuation = STANDARD.encode(&bytes); + use sha2::Digest; + let mut digest = sha2::Sha256::new(); + digest.update(forged.source_id.as_bytes()); + digest.update(forged.scope_id.as_bytes()); + digest.update(forged.generation.to_le_bytes()); + digest.update(forged.anchor.height.to_le_bytes()); + digest.update(forged.anchor.hash.as_bytes()); + digest.update([u8::from(forged.complete)]); + digest.update(&bytes); + forged.checkpoint_id = crate::web::to_hex(&digest.finalize()); + assert_ne!( + forged.checkpoint_id, + previous.as_ref().unwrap().checkpoint_id + ); + let mut attacker = Pass::new( + scope.clone(), + Some(forged), + Limits { + max_exchanges: 256, + ..Limits::default() + }, + ) + .unwrap(); + let owned_checkpoint = wallet.transparent_pir_checkpoint().unwrap(); + let owned_rows = rows(&wallet); + let mut rejected = false; + let mut chain_client = zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient::new(fixture.channel.clone()); + for _ in 0..2000 { + match attacker.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => attacker + .accept(request_id, answer(server, operation)) + .unwrap(), + Step::NeedChain { height, .. } => { + let point = runtime + .block_on(chain_client.get_tree_state(BlockId { + height, + hash: vec![], + })) + .unwrap() + .into_inner(); + attacker.accept_chain(height, point.hash).unwrap(); + } + Step::Checkpoint { checkpoint } => { + let json = serde_json::to_string(&checkpoint).unwrap(); + let error = match attacker.accepted_rewind().unwrap() { + Some(witness) => wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut native, + &json, + witness, + ) + .unwrap_err(), + None => wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &json, + ) + .unwrap_err(), + }; + assert_eq!(error.to_string(), "transparent_pir_checkpoint_invalid"); + assert_eq!( + wallet.transparent_pir_checkpoint().unwrap(), + owned_checkpoint + ); + assert!( + rows(&wallet) == owned_rows, + "owned wallet state changed unexpectedly" + ); + rejected = true; + break; + } + Step::Stopped { .. } => { + panic!("forged continuation must reach owner rejection") + } + } + } + assert!(rejected); + } + let mut pass = Pass::new( + scope, + previous.clone(), + Limits { + max_exchanges: 256, + ..Limits::default() + }, + ) + .unwrap(); + let mut client=zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient::new(fixture.channel.clone()); + let mut reached = false; + for _ in 0..2000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => pass.accept(request_id, answer(server, operation)).unwrap(), + Step::NeedChain { height, .. } => { + let point = runtime + .block_on(client.get_tree_state(BlockId { + height, + hash: vec![], + })) + .unwrap() + .into_inner(); + assert_eq!(point.height, height); + assert_eq!(point.network, "regtest"); + assert_eq!(point.hash, hash(height).to_display_hex()); + if height == 1 { + old_point_reads += 1; + } + observed.insert(height, point.hash.clone()); + pass.accept_chain(height, point.hash).unwrap(); + } + Step::Checkpoint { checkpoint } => { + if let Some(a) = checkpoint.rewind_to.as_ref() { + reached = true; + assert!(phase > 0); + assert_eq!(a.height, 1); + assert_eq!(a.hash, observed[&1]); + assert!(a.height < birthday); + assert!(missing); + eprintln!("SAFE_CENSUS phase={} ancestor_height={} birthday={} db_hash_absent={} independently_observed={} cached_without_second_read={}",phase,a.height,birthday,missing,true,old_point_reads==1); + } + if phase == 1 && checkpoint.rewind_to.is_some() { + let calls = fixture.calls.lock().unwrap().len(); + pass.finish_budget().unwrap(); + assert_eq!(fixture.calls.lock().unwrap().len(), calls); + } + let json = serde_json::to_string(&checkpoint).unwrap(); + let before = wallet.transparent_pir_checkpoint().unwrap(); + if checkpoint.rewind_to.is_some() { + let error = wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &json, + ) + .unwrap_err(); + assert_eq!(error.to_string(), "transparent_pir_rewind_anchor_unknown"); + assert!( + wallet.transparent_pir_checkpoint().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + } + if checkpoint.rewind_to.is_some() { + // Existing canonical evidence always wins over the witness fallback. + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + conn.execute("INSERT INTO blocks (height,hash,time,sapling_tree) VALUES (1,?1,0,X'000000')",[vec![0xee;32]]).unwrap(); + let error = wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut native, + &json, + pass.accepted_rewind().unwrap().unwrap(), + ) + .unwrap_err(); + assert_eq!(error.to_string(), "transparent_pir_rewind_anchor_unknown"); + assert!( + wallet.transparent_pir_checkpoint().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + conn.execute("DELETE FROM blocks WHERE height=1", []) + .unwrap(); + drop(conn); + let mut changed = checkpoint.clone(); + changed.pending_pages += 1; + let witness = pass.accepted_rewind().unwrap().unwrap(); + let error = wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut native, + &serde_json::to_string(&changed).unwrap(), + witness, + ) + .unwrap_err(); + assert_eq!(error.to_string(), "transparent_pir_rewind_evidence_invalid"); + assert!( + wallet.transparent_pir_checkpoint().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + let mut cancelled = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + cancelled.cancel(); + let witness = pass.accepted_rewind().unwrap().unwrap(); + let error = wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut cancelled, + &json, + witness, + ) + .unwrap_err(); + assert_eq!(error.to_string(), "native_recovery_cancelled"); + assert!( + wallet.transparent_pir_checkpoint().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + } + let result = match pass.accepted_rewind().unwrap() { + Some(witness) => wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut native, + &json, + witness, + ), + None => wallet.apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &json, + ), + }; + assert!(result.is_ok(),"valid independently observed unchanged-chain historical rewind must apply: {}",result.err().map(|e|e.to_string()).unwrap_or_default()); + let applied = wallet.transparent_pir_checkpoint().unwrap(); + let applied_rows = rows(&wallet); + match pass.accepted_rewind().unwrap() { + Some(witness) => wallet + .apply_transparent_pir_checkpoint_with_source_and_rewind( + &fixture.source, + &mut native, + &json, + witness, + ) + .unwrap(), + None => wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut native, + &json, + ) + .unwrap(), + }; + assert!( + wallet.transparent_pir_checkpoint().unwrap() == applied, + "owned wallet state changed unexpectedly" + ); + assert!( + rows(&wallet) == applied_rows, + "owned wallet state changed unexpectedly" + ); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + previous = Some(checkpoint); + } + Step::Stopped { status } => { + if phase == 1 { + assert_eq!( + status, + crate::transparent_pir::acquisition::Status::Incomplete { + reason: "budget".into() + } + ); + } else { + assert_eq!( + status, + crate::transparent_pir::acquisition::Status::Complete + ); + } + break; + } + } + } + if phase == 1 || phase == 3 { + assert!(reached, "replacement must produce historical rewind"); + } + assert_eq!( + old_point_reads, 1, + "cold and moving-anchor lineage reuse the authenticated old historical point" + ); + } +} + +// Authenticated fixture RPC is the chain boundary. The callback itself may not +// perform any RPC; this also models the caller's no-I/O checkpoint drain. +fn drive_observed_rewind( + scope: Scope, + previous: Option, + server: &service::InProcess, + fixture: &Fixture, + runtime: &tokio::runtime::Runtime, + mut apply: impl FnMut(&Pass, &Checkpoint), +) -> (Checkpoint, Vec) { + let mut pass = Pass::new( + scope, + previous, + Limits { + max_exchanges: 256, + ..Limits::default() + }, + ) + .unwrap(); + let mut last = None; + let mut reads = Vec::new(); + let mut client = zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient::new(fixture.channel.clone()); + for _ in 0..2000 { + match pass.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => pass.accept(request_id, answer(server, operation)).unwrap(), + Step::NeedChain { height, .. } => { + let point = runtime + .block_on(client.get_tree_state(BlockId { + height, + hash: vec![], + })) + .unwrap() + .into_inner(); + assert_eq!(point.height, height); + assert_eq!(point.network, "regtest"); + reads.push(height); + pass.accept_chain(height, point.hash).unwrap(); + } + Step::Checkpoint { checkpoint } => { + let before = fixture.calls.lock().unwrap().len(); + apply(&pass, &checkpoint); + assert_eq!( + fixture.calls.lock().unwrap().len(), + before, + "mint/apply must perform no I/O" + ); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + last = Some(checkpoint); + } + Step::Stopped { status } => { + assert_eq!( + status, + crate::transparent_pir::acquisition::Status::Complete + ); + return (last.unwrap(), reads); + } + } + } + panic!("bounded observed-rewind fixture did not complete") +} + +#[test] +fn historical_rewind_web_candidate_and_cold_replay_remain_scoped() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let fixture = runtime.block_on(Fixture::new_with_tip(root.path(), 4)); + let account = account_from_mnemonic(PUBLIC_SEED, Network::Regtest, 0).unwrap(); + let mut wallet = crate::web::WebWallet::from_account(account, 3).unwrap(); + let mut client = zcash_client_backend::proto::service::compact_tx_streamer_client::CompactTxStreamerClient::new(fixture.channel.clone()); + let tree = runtime + .block_on(client.get_tree_state(BlockId { + height: 2, + hash: vec![], + })) + .unwrap() + .into_inner(); + wallet.apply_tree_state_json(&serde_json::json!({"network":tree.network,"height":tree.height,"hash":tree.hash,"saplingTree":tree.sapling_tree,"orchardTree":tree.orchard_tree,"ironwoodTree":tree.ironwood_tree}).to_string()).unwrap(); + wallet + .apply_compact_blocks_blob(&crate::web::encode_delimited(vec![ + block(3, Network::Regtest.genesis_hash()), + block(4, Network::Regtest.genesis_hash()), + ])) + .unwrap(); + let genesis = BlockHash::from_display_hex(Network::Regtest.genesis_hash()).unwrap(); + let hash = |height| { + if height == 0 { + genesis + } else { + BlockHash::from_display_hex(&format!("{:02x}", height as u8).repeat(32)).unwrap() + } + }; + let publication = tempfile::tempdir().unwrap(); + let first = service::publish_for_network( + publication.path(), + &[ + service::ShardSpec::first(0, 1, true, vec![]), + service::ShardSpec::first(2, 4, false, vec![]), + ], + service::SEAL, + hash, + vec![], + "regtest", + genesis, + ); + let old_server = service::InProcess::serve(publication.path()); + let replacement = tempfile::tempdir().unwrap(); + let mut tail = service::ShardSpec::first(2, 4, false, vec![]); + tail.revision = 1; + tail.supersedes = first.shards[1].manifest_digest.clone(); + service::publish_for_network( + replacement.path(), + &[service::ShardSpec::first(0, 1, true, vec![]), tail], + service::SEAL, + hash, + vec![], + "regtest", + genesis, + ); + let next_server = service::InProcess::serve(replacement.path()); + let mut previous = None; + let mut historic_reads = 0; + for phase in 0..3 { + wallet = crate::web::WebWallet::from_snapshot(&wallet.to_snapshot().unwrap()).unwrap(); + let scope = + serde_json::from_str(&wallet.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let expected_replay = (phase == 2).then(|| previous.clone().unwrap()); + let mut saw_rewind = false; + let (last, reads) = drive_observed_rewind( + scope, + previous, + if phase == 0 { + &old_server + } else { + &next_server + }, + &fixture, + &runtime, + |pass, checkpoint| { + let json = serde_json::to_string(checkpoint).unwrap(); + let before = wallet.to_snapshot().unwrap(); + if checkpoint.rewind_to.is_some() { + saw_rewind = true; + assert!(phase > 0); + assert_eq!(checkpoint.rewind_to.as_ref().unwrap().height, 1); + assert_eq!( + wallet + .apply_transparent_pir_checkpoint(&json) + .unwrap_err() + .to_string(), + "transparent_pir_rewind_anchor_unknown" + ); + assert!( + wallet.to_snapshot().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + // A different scope cannot supply an authority token for this owner. + // Only a missing canonical point admits the fallback. A known + // conflict or malformed stored point must never be replaced. + for canonical in ["ee".repeat(32), "malformed".into()] { + wallet.block_hashes.insert(1, canonical); + let conflicting = wallet.to_snapshot().unwrap(); + assert_eq!( + wallet + .apply_transparent_pir_checkpoint_with_rewind( + &json, + pass.accepted_rewind().unwrap().unwrap() + ) + .unwrap_err() + .to_string(), + "transparent_pir_rewind_anchor_unknown" + ); + assert!( + wallet.to_snapshot().unwrap() == conflicting, + "owned wallet state changed unexpectedly" + ); + wallet.block_hashes.remove(&1); + } + let mut changed = checkpoint.clone(); + changed.source_id = "cd".repeat(32); + assert_eq!( + wallet + .apply_transparent_pir_checkpoint_with_rewind( + &serde_json::to_string(&changed).unwrap(), + pass.accepted_rewind().unwrap().unwrap() + ) + .unwrap_err() + .to_string(), + "transparent_pir_rewind_evidence_invalid" + ); + assert!( + wallet.to_snapshot().unwrap() == before, + "owned wallet state changed unexpectedly" + ); + } + let (candidate, _) = match pass.accepted_rewind().unwrap() { + Some(witness) => wallet + .transparent_pir_checkpoint_snapshot_with_rewind(&json, witness) + .unwrap(), + None => wallet.transparent_pir_checkpoint_snapshot(&json).unwrap(), + }; + assert!( + wallet.to_snapshot().unwrap() == before, + "candidate must not publish to live wallet" + ); + let cold_candidate = crate::web::WebWallet::from_snapshot(&candidate).unwrap(); + assert!( + cold_candidate.transparent_pir_checkpoint().as_deref() == Some(json.as_str()), + "candidate lost its exact checkpoint" + ); + match pass.accepted_rewind().unwrap() { + Some(witness) => wallet + .apply_transparent_pir_checkpoint_with_rewind(&json, witness) + .unwrap(), + None => wallet.apply_transparent_pir_checkpoint(&json).unwrap(), + }; + let applied = wallet.to_snapshot().unwrap(); + match pass.accepted_rewind().unwrap() { + Some(witness) => wallet + .apply_transparent_pir_checkpoint_with_rewind(&json, witness) + .unwrap(), + None => wallet.apply_transparent_pir_checkpoint(&json).unwrap(), + }; + assert!( + wallet.to_snapshot().unwrap() == applied, + "identical full context is an idempotent owned replay" + ); + wallet + .acknowledge_transparent_pir_checkpoint(&checkpoint.checkpoint_id) + .unwrap(); + }, + ); + if let Some(expected) = expected_replay { + // Reopening may normalize opaque acquisition journals. It must keep + // the exact public facts and reuse the historical chain observation. + let mut public = last.clone(); + public.checkpoint_id = expected.checkpoint_id.clone(); + public.previous_checkpoint_id = expected.previous_checkpoint_id.clone(); + public.continuation = expected.continuation.clone(); + assert!( + public == expected, + "cold resume changed public checkpoint facts" + ); + } + if phase == 1 { + assert!(saw_rewind); + } + historic_reads += reads.iter().filter(|height| **height == 1).count(); + assert_eq!( + historic_reads, 1, + "cold exact-anchor replay reuses the previously observed historical point" + ); + previous = Some(last); + } +} diff --git a/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs b/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs new file mode 100644 index 0000000..21f63b6 --- /dev/null +++ b/crates/z-engine/src/native/wallet/transparent_pir_source_tests.rs @@ -0,0 +1,1065 @@ +use super::*; +use crate::transparent_pir::{TransparentPirCheckpoint, TransparentPirRecord, TransparentPirScope}; +use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; +use z_transparent_pir_v2::Coverage; + +fn checkpoint( + scope: TransparentPirScope, + script: &str, + events: Vec, + previous: Option, + id: u8, +) -> TransparentPirCheckpoint { + TransparentPirCheckpoint { + source_id: scope.source_id, + generation: scope.generation, + scope_id: scope.scope_id, + anchor: scope.anchor.clone(), + checkpoint_id: format!("{id:02x}").repeat(32), + previous_checkpoint_id: previous, + covered_through: scope.anchor.height, + complete: true, + pending_pages: 0, + unresolved_spends: 0, + records: events + .into_iter() + .map(|e| TransparentPirRecord { + script: script.into(), + event: crate::web::to_hex(&e.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }) + .collect(), + coverage: scope + .scripts + .into_iter() + .map(|s| Coverage { + script: s.script, + required_from: s.required_from, + through: scope.anchor.height, + revision: "ef".repeat(32), + }) + .collect(), + rewind_to: None, + continuation: String::new(), + } +} +fn receive() -> TransparentEvent { + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x11; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }) +} +fn spend() -> TransparentEvent { + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid([0x22; 32]), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([0x11; 32]), + spent_output_index: 0, + }) +} + +#[tokio::test] +async fn native_transparent_pass_drop_does_not_commit_or_leave_a_pending_session() { + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let source_id = "ab".repeat(32); + let cancel = Arc::new(RecoveryCancellation::new()); + let pass = wallet + .begin_transparent_pir_with_source(&fixture.source, &source_id, cancel.clone()) + .unwrap(); + let original_scope = pass.scope_json().unwrap(); + assert!(database_lease::exclusive(&wallet.paths.data_db).is_err()); + drop(pass); + drop(database_lease::exclusive(&wallet.paths.data_db).unwrap()); + assert!(!cancel.is_cancelled()); + assert!(wallet.transparent_pir_checkpoint().unwrap().is_none()); + assert!(wallet.history(20).unwrap().is_empty()); + + // A budget/transport abandonment can begin again with the same live token. + let pass = wallet + .begin_transparent_pir_with_source(&fixture.source, &source_id, cancel.clone()) + .unwrap(); + assert!( + pass.scope_json().unwrap() == original_scope, + "abandoned scope must be reusable" + ); + cancel.cancel(); + drop(pass); + assert!(wallet.transparent_pir_checkpoint().unwrap().is_none()); + assert!(wallet.history(20).unwrap().is_empty()); + + // Explicit cancellation poisons that token, not the durable wallet session. + let mut restarted = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &source_id, + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: TransparentPirScope = + serde_json::from_str(&restarted.scope_json().unwrap()).unwrap(); + let script = scope.scripts[0].script.clone(); + let complete = checkpoint(scope, &script, vec![], None, 0x71); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut restarted, + &serde_json::to_string(&complete).unwrap(), + ) + .unwrap(); + drop(restarted); + let enrolled = wallet.transparent_pir_status_for(&source_id).unwrap(); + assert_eq!(enrolled["completion"], "complete-for-enrolled-scope"); + assert_eq!(enrolled["checkpointId"], complete.checkpoint_id); + let other_source = "cd".repeat(32); + let masked = wallet.transparent_pir_status_for(&other_source).unwrap(); + assert_eq!(masked["completion"], "incomplete"); + assert!(masked.get("scope").is_none()); + assert!(masked.get("confirmedZat").is_none()); + assert!(wallet + .begin_transparent_pir_with_source( + &fixture.source, + &other_source, + Arc::new(RecoveryCancellation::new()), + ) + .err() + .unwrap() + .to_string() + .contains("transparent_pir_source_changed")); + assert_eq!( + wallet.transparent_pir_status_for(&source_id).unwrap(), + enrolled + ); +} + +#[tokio::test] +async fn native_transparent_pir_canonical_receive_spend_reopen_and_cancel() { + use zcash_keys::encoding::AddressCodec; + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let address = transparent::address::TransparentAddress::decode( + &wallet.network, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let script = crate::web::to_hex(&script.0 .0); + let cancel = Arc::new(RecoveryCancellation::new()); + let mut pass = wallet + .begin_transparent_pir_with_source(&fixture.source, &"ab".repeat(32), cancel) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let mut c = checkpoint(scope, &script, vec![receive()], None, 1); + c.complete = false; + c.pending_pages = 1; + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c).unwrap(), + ) + .unwrap(); + let conn = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let (value,raw):(u64,Option>)=conn.query_row("SELECT n.value_zat,t.raw FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1",[[0x11u8;32].as_slice()],|r|Ok((r.get(0)?,r.get(1)?))).unwrap(); + assert_eq!(value, 150000); + assert!(raw.is_none()); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + // PIR rows live in the canonical DB, but this source-bound wallet cannot + // enter ordinary transparent shielding or direct endpoint send routines. + let calls_before = fixture.calls.lock().unwrap().len(); + code( + wallet.shield(&auth(), 1).await, + "native_wallet_source_required", + ); + code(wallet.max_send(None), "native_wallet_source_required"); + code( + wallet.send(&auth(), "irrelevant", 1, None).await, + "native_wallet_source_required", + ); + assert_eq!(fixture.calls.lock().unwrap().len(), calls_before); + conn.execute( + "UPDATE transparent_received_outputs SET lock_expiry_height=100 WHERE value_zat=150000", + [], + ) + .unwrap(); + // Reapplying identical evidence must not erase a reservation. + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c).unwrap(), + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let c2 = checkpoint( + scope, + &script, + vec![receive(), spend()], + Some(c.checkpoint_id.clone()), + 2, + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c2).unwrap(), + ) + .unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 1 + ); + assert_eq!(wallet.history(20).unwrap().len(), 2); + assert_eq!( + wallet.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + drop(pass); + drop(wallet); + let reopened = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + assert_eq!(reopened.history(20).unwrap().len(), 2); + let mut recut = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&recut.scope_json().unwrap()).unwrap(); + let mut c3 = checkpoint( + scope, + &script, + vec![receive()], + Some(c2.checkpoint_id.clone()), + 3, + ); + c3.rewind_to = Some(crate::transparent_pir::TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut recut, + &serde_json::to_string(&c3).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + drop(recut); + let token = Arc::new(RecoveryCancellation::new()); + let mut pass = reopened + .begin_transparent_pir_with_source(&fixture.source, &"ab".repeat(32), token.clone()) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let c4 = checkpoint(scope, &script, vec![receive()], Some(c3.checkpoint_id), 4); + token.cancel(); + assert!(reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&c4).unwrap() + ) + .is_err()); + assert_eq!( + reopened + .transparent_pir_checkpoint() + .unwrap() + .map(|s| serde_json::from_str::(&s) + .unwrap() + .checkpoint_id), + Some("03".repeat(32)) + ); + drop(pass); + let mut mined = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&mined.scope_json().unwrap()).unwrap(); + let c5 = checkpoint( + scope, + &script, + vec![receive(), spend()], + Some("03".repeat(32)), + 5, + ); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut mined, + &serde_json::to_string(&c5).unwrap(), + ) + .unwrap(); + drop(mined); + reopened + .open_db() + .unwrap() + .truncate_to_height(1.into()) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + let saved = reopened.transparent_pir_checkpoint().unwrap(); + code( + reopened.begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ), + "transparent_pir_rewind_required", + ); + assert!( + reopened.transparent_pir_checkpoint().unwrap() == saved, + "lower target must retain private acquisition" + ); + fixture.mode.store(4, Ordering::Release); + reopened + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let mut after_reorg = reopened + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: TransparentPirScope = + serde_json::from_str(&after_reorg.scope_json().unwrap()).unwrap(); + assert_eq!(scope.anchor.height, 2); + let mut partial = checkpoint(scope.clone(), &script, vec![], Some(c5.checkpoint_id), 6); + partial.rewind_to = Some(crate::transparent_pir::TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + partial.complete = false; + partial.pending_pages = 1; + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut after_reorg, + &serde_json::to_string(&partial).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["completion"], + "incomplete" + ); + let c6 = checkpoint( + scope, + &script, + vec![receive()], + Some(partial.checkpoint_id), + 7, + ); + reopened + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut after_reorg, + &serde_json::to_string(&c6).unwrap(), + ) + .unwrap(); + assert_eq!( + reopened.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM transparent_received_output_spends", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 0 + ); + assert_eq!( + conn.query_row( + "SELECT lock_expiry_height FROM transparent_received_outputs WHERE value_zat=150000", + [], + |r| r.get::<_, u32>(0) + ) + .unwrap(), + 100 + ); + assert!(!fixture.calls.lock().unwrap().iter().any(|m| matches!( + m.as_str(), + "GetTransaction" | "GetAddressUtxos" | "SendTransaction" + ))); +} + +#[tokio::test] +async fn native_transparent_pir_amount_counts_only_pir_recovered_receives() { + use zcash_keys::encoding::AddressCodec; + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let address = transparent::address::TransparentAddress::decode( + &wallet.network, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + { + let mut db = wallet.open_db().unwrap(); + let account = NativeWallet::primary_account_id(&db).unwrap(); + for (txid, value) in [([0x11; 32], 150000), ([0x33; 32], 40000)] { + let output = zcash_client_backend::wallet::WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(txid, 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(value).unwrap(), + script.clone(), + ), + Some(1.into()), + Some(account), + None, + None, + ) + .unwrap(); + db.put_received_transparent_utxo(&output).unwrap(); + } + } + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + let mut pass = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: TransparentPirScope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let script = crate::web::to_hex(&script.0 .0); + let empty = checkpoint(scope.clone(), &script, vec![], None, 0x81); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&empty).unwrap(), + ) + .unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + let recovered = checkpoint( + scope.clone(), + &script, + vec![receive()], + Some(empty.checkpoint_id), + 0x82, + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&recovered).unwrap(), + ) + .unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "150000" + ); + let spent = checkpoint( + scope, + &script, + vec![receive(), spend()], + Some(recovered.checkpoint_id), + 0x83, + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&spent).unwrap(), + ) + .unwrap(); + assert_eq!( + wallet.transparent_pir_status().unwrap()["confirmedZat"], + "0" + ); + let db = rusqlite::Connection::open(&wallet.paths.data_db).unwrap(); + let ordinary:u64 = db.query_row("SELECT n.value_zat FROM transparent_received_outputs n JOIN transactions t ON t.id_tx=n.transaction_id WHERE t.txid=?1", [[0x33u8;32].as_slice()], |r|r.get(0)).unwrap(); + assert_eq!(ordinary, 40000); +} + +#[tokio::test] +async fn native_pir_only_funding_is_excluded_from_real_shielding_proposal() { + use zcash_keys::encoding::AddressCodec; + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let address = transparent::address::TransparentAddress::decode( + &wallet.network, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let mut pass = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope: TransparentPirScope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let cp = checkpoint( + scope, + &crate::web::to_hex(&script.0 .0), + vec![receive()], + None, + 0x91, + ); + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&cp).unwrap(), + ) + .unwrap(); + drop(pass); + let mut db = wallet.open_db().unwrap(); + let account = NativeWallet::primary_account_id(&db).unwrap(); + let control = zcash_client_backend::wallet::WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([0x33; 32], 0), + transparent::bundle::TxOut::new(Zatoshis::from_u64(40000).unwrap(), script.clone()), + Some(1.into()), + Some(account), + None, + None, + ) + .unwrap(); + db.put_received_transparent_utxo(&control).unwrap(); + drop(db); + assert_eq!( + native_funding_selection(&wallet, address).unwrap(), + vec![[0x33; 32]], + "PIR-only funding must not enter actual shielding selection alongside independent control" + ); + // Public source binding remains a separate, earlier spending boundary. + code( + wallet.shield(&auth(), 1).await, + "native_wallet_source_required", + ); + assert_eq!( + wallet.transparent_pir_status_for(&"ab".repeat(32)).unwrap()["confirmedZat"], + "150000" + ); + + // Explicit transparent SpendPolicy and decoding a proposal do not bypass the + // funding boundary; the create paths repeat this check in their write txn. + let mut db = wallet.open_db().unwrap(); + let request = TransactionRequest::new(vec![zip_payment( + &wallet.unified_address().unwrap(), + 100000, + None, + ) + .unwrap()]) + .unwrap(); + let policy=shielded_spend_policy().with_transparent(zcash_client_backend::data_api::wallet::input_selection::TransparentSpendPolicy::any_account_addr()); + let result = db.transactionally_with_extension(|wdb, ext| -> anyhow::Result<_> { + wallet + .propose_fee_in_snapshot(wdb, ext, request, policy) + .map_err(anyhow::Error::new) + }); + assert!(result + .unwrap_err() + .to_string() + .contains("transparent_pir_funding_unverified")); + let raw_proposal = + zcash_client_backend::data_api::wallet::propose_shielding::<_, _, _, _, Infallible>( + &mut db, + &wallet.network, + &GreedyInputSelector::new(), + &SingleOutputChangeStrategy::new( + StandardFeeRule::Zip317, + None, + ShieldedPool::Orchard, + DustOutputPolicy::default(), + ), + Zatoshis::from_u64(1).unwrap(), + &[address], + account, + shield_confirmations(wallet.network), + CoinbaseFilter::AllTransparentOutputs, + None, + ) + .unwrap(); + let result = db.transactionally_with_extension(|_, ext| { + super::super::transparent_pir_funding::check_proposal(ext, &raw_proposal) + }); + assert!(result + .unwrap_err() + .to_string() + .contains("transparent_pir_funding_unverified")); + + // Locking the independent output cannot make the selector fall back to PIR. + use zcash_client_backend::{ + data_api::OutputLockStore, + wallet::{LockOwner, OutputRef}, + }; + let owner = LockOwner::new([0x55; 32]); + let reference = OutputRef::new( + TxId::from_bytes([0x33; 32]), + zcash_protocol::PoolType::Transparent, + 0, + ); + db.lock_outputs(&[reference], owner, 100.into()).unwrap(); + assert!(native_funding_selection(&wallet, address).is_err()); + assert!( + db.unlock_output(&reference, owner).unwrap(), + "selection must not release another flow's reservation" + ); + drop(db); + + // Re-observation alone never grants the native promotion authority this + // version does not implement. Dropping enrollment and reopening does not either. + let pir_output = zcash_client_backend::wallet::WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([0x11; 32], 0), + transparent::bundle::TxOut::new(Zatoshis::from_u64(150000).unwrap(), script.clone()), + Some(1.into()), + Some(account), + None, + None, + ) + .unwrap(); + wallet + .open_db() + .unwrap() + .put_received_transparent_utxo(&pir_output) + .unwrap(); + drop(wallet); + let cold = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + assert_eq!( + native_funding_selection(&cold, address).unwrap(), + vec![[0x33; 32]] + ); + code( + cold.shield(&auth(), 1).await, + "native_wallet_source_required", + ); + let ordinary_open = NativeWallet::open(root.path()).unwrap(); + code( + ordinary_open.shield(&auth(), 1).await, + "native_wallet_source_required", + ); + drop(ordinary_open); + + // Absent/v1 funding provenance cannot establish sibling authority. + // The native owner durably quarantines ambiguous legacy canonical funds. + let conn = rusqlite::Connection::open(&cold.paths.data_db).unwrap(); + let original: Vec = conn + .query_row("SELECT state FROM ext_transparent_pir_v1", [], |r| r.get(0)) + .unwrap(); + conn.execute("DROP TABLE ext_transparent_pir_funding_guard_v1", []) + .unwrap(); + assert!(native_funding_selection(&cold, address).is_err()); + cold.transparent_pir_status().unwrap(); // native owner reinstates the durable guard + assert_eq!( + native_funding_selection(&cold, address).unwrap(), + vec![[0x33; 32]] + ); + + for (version, missing) in [(None, false), (None, true), (Some(1), false)] { + let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap(); + legacy + .as_object_mut() + .unwrap() + .remove("nativeFundingVersion"); + if let Some(v) = version { + legacy["nativeFundingVersion"] = serde_json::json!(v); + } + legacy["unverifiedOutputs"] = if missing { + serde_json::Value::Null + } else { + serde_json::json!([]) + }; + conn.execute( + "UPDATE ext_transparent_pir_v1 SET state=?1", + [serde_json::to_vec(&legacy).unwrap()], + ) + .unwrap(); + cold.transparent_pir_status().unwrap(); // owner commits the conservative migration + assert!(native_funding_selection(&cold, address).is_err()); + let saved: Vec = conn + .query_row("SELECT state FROM ext_transparent_pir_v1", [], |r| r.get(0)) + .unwrap(); + let saved: serde_json::Value = serde_json::from_slice(&saved).unwrap(); + assert_eq!(saved["nativeFundingVersion"], 2); + assert_eq!(saved["unverifiedOutputs"].as_array().unwrap().len(), 2); + } + let mut invalid: serde_json::Value = serde_json::from_slice(&original).unwrap(); + invalid["nativeFundingVersion"] = 3.into(); + conn.execute( + "UPDATE ext_transparent_pir_v1 SET state=?1", + [serde_json::to_vec(&invalid).unwrap()], + ) + .unwrap(); + assert!(native_funding_selection(&cold, address).is_err()); + invalid["nativeFundingVersion"] = 2.into(); + invalid["unverifiedOutputs"] = serde_json::Value::Null; + conn.execute( + "UPDATE ext_transparent_pir_v1 SET state=?1", + [serde_json::to_vec(&invalid).unwrap()], + ) + .unwrap(); + assert!(native_funding_selection(&cold, address).is_err()); + conn.execute("UPDATE ext_transparent_pir_v1 SET state=?1", [original]) + .unwrap(); + drop(conn); + + // A real source checkpoint retirement removes deletion ownership, while + // funding quarantine survives even if the canonical outpoint is reinserted. + let mut pass = cold + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let mut retired = checkpoint( + scope, + &crate::web::to_hex(&script.0 .0), + vec![], + Some(cp.checkpoint_id), + 0x92, + ); + retired.rewind_to = Some(crate::transparent_pir::TransparentPirAnchor { + height: 0, + hash: cold.network.genesis_hash().into(), + }); + cold.apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&retired).unwrap(), + ) + .unwrap(); + drop(pass); + cold.open_db() + .unwrap() + .put_received_transparent_utxo(&pir_output) + .unwrap(); + assert_eq!( + native_funding_selection(&cold, address).unwrap(), + vec![[0x33; 32]] + ); + let conn = rusqlite::Connection::open(&cold.paths.data_db).unwrap(); + let bytes: Vec = conn + .query_row("SELECT state FROM ext_transparent_pir_v1", [], |r| r.get(0)) + .unwrap(); + let state: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(state["projectedOutputs"].as_array().unwrap().is_empty()); + assert!(state["unverifiedOutputs"] + .as_array() + .unwrap() + .contains(&serde_json::json!(format!("r:{}:0", "11".repeat(32))))); + + // The durable receipt fixture contains serialized disposable bytes, not a + // real submission. Its lifecycle and bytes must survive funding refusal. + let (receipt_id, receipt_raw) = saved_transaction(&cold); + let receipt_before = serde_json::to_value(cold.payment_receipt(&receipt_id).unwrap()).unwrap(); + + // Losing or corrupting durable provenance fails closed, preserving rows. + conn.execute("UPDATE ext_transparent_pir_v1 SET state=X'00'", []) + .unwrap(); + assert!(native_funding_selection(&cold, address).is_err()); + conn.execute("DELETE FROM ext_transparent_pir_v1", []) + .unwrap(); + assert!(native_funding_selection(&cold, address).is_err()); + assert!(cold.transparent_pir_status().is_err()); + let outputs: u64 = conn + .query_row( + "SELECT COUNT(*) FROM transparent_received_outputs", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(outputs, 3); + assert_eq!( + serde_json::to_value(cold.payment_receipt(&receipt_id).unwrap()).unwrap(), + receipt_before + ); + let retained: Vec = conn + .query_row( + "SELECT raw FROM transactions WHERE created IS NOT NULL", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + retained == receipt_raw, + "funding checks must retain saved transaction bytes" + ); +} + +fn native_funding_selection( + wallet: &NativeWallet, + address: transparent::address::TransparentAddress, +) -> anyhow::Result> { + let mut db = wallet.open_db()?; + let account = NativeWallet::primary_account_id(&db)?; + let proposal = db.transactionally_with_extension(|wdb, ext| -> anyhow::Result<_> { + Ok(wallet.propose_shielding_in_snapshot( + wdb, + ext, + &[address], + account, + Zatoshis::from_u64(1).unwrap(), + )?) + })?; + Ok(proposal + .steps() + .iter() + .flat_map(|s| s.transparent_inputs()) + .map(|o| *o.outpoint().hash()) + .collect()) +} + +async fn pending_sibling_confirmation_case(split: bool) { + use zcash_keys::encoding::AddressCodec; + let mut observed = Vec::new(); + { + let root = tempfile::tempdir().unwrap(); + let fixture = Fixture::new(root.path()).await; + let wallet = fixture.restore(root.path()).await; + wallet + .sync_with_wallet_source_reported_cancellable( + &fixture.source, + None, + Arc::new(AtomicBool::new(false)), + ) + .await + .unwrap(); + let address = transparent::address::TransparentAddress::decode( + &wallet.network, + &wallet.transparent_address().unwrap().unwrap(), + ) + .unwrap(); + let script: transparent::address::Script = address.script().into(); + let mut db = wallet.open_db().unwrap(); + let account = NativeWallet::primary_account_id(&db).unwrap(); + for (id, index, value, height) in [ + (0x33, 0, 40000, Some(1.into())), + (0x44, 0, 60000, None), + (0x44, 1, 70000, None), + ] { + let output = zcash_client_backend::wallet::WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([id; 32], index), + transparent::bundle::TxOut::new(Zatoshis::from_u64(value).unwrap(), script.clone()), + height, + Some(account), + None, + None, + ) + .unwrap(); + db.put_received_transparent_utxo(&output).unwrap(); + } + drop(db); + let records = vec![ + receive(), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x33; 32]), + transaction_index: 1, + output_index: 0, + value: 40000, + coinbase: false, + }), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x33; 32]), + transaction_index: 1, + output_index: 1, + value: 50000, + coinbase: false, + }), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x44; 32]), + transaction_index: 1, + output_index: 0, + value: 60000, + coinbase: false, + }), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([0x44; 32]), + transaction_index: 1, + output_index: 1, + value: 70000, + coinbase: false, + }), + ]; + let mut pass = wallet + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let first = if split { + records[..4].to_vec() + } else { + records.clone() + }; + let mut cp = checkpoint(scope, &crate::web::to_hex(&script.0 .0), first, None, 0xa1); + if split { + cp.complete = false; + cp.pending_pages = 1; + } + wallet + .apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&cp).unwrap(), + ) + .unwrap(); + drop(pass); + observed.push(native_funding_selection(&wallet, address).unwrap()); + drop(wallet); + let cold = NativeWallet::open_with_wallet_source(root.path(), &fixture.source).unwrap(); + observed.push(native_funding_selection(&cold, address).unwrap()); + code( + cold.shield(&auth(), 1).await, + "native_wallet_source_required", + ); + if split { + let mut pass = cold + .begin_transparent_pir_with_source( + &fixture.source, + &"ab".repeat(32), + Arc::new(RecoveryCancellation::new()), + ) + .unwrap(); + let scope = serde_json::from_str(&pass.scope_json().unwrap()).unwrap(); + let next = checkpoint( + scope, + &crate::web::to_hex(&script.0 .0), + records, + Some(cp.checkpoint_id), + 0xa2, + ); + cold.apply_transparent_pir_checkpoint_with_source( + &fixture.source, + &mut pass, + &serde_json::to_string(&next).unwrap(), + ) + .unwrap(); + drop(pass); + observed.push(native_funding_selection(&cold, address).unwrap()); + } + } + assert_eq!(observed,vec![vec![[0x33;32]];if split {3}else{2}],"publisher metadata cannot confer funding authority on pending siblings in hot, cold, or later checkpoint stages"); +} + +#[tokio::test] +async fn native_pir_same_checkpoint_pending_siblings_remain_read_only() { + pending_sibling_confirmation_case(false).await; +} +#[tokio::test] +async fn native_pir_cross_checkpoint_pending_siblings_remain_read_only() { + pending_sibling_confirmation_case(true).await; +} diff --git a/crates/z-engine/src/transparent_pir.rs b/crates/z-engine/src/transparent_pir.rs new file mode 100644 index 0000000..df4a15b --- /dev/null +++ b/crates/z-engine/src/transparent_pir.rs @@ -0,0 +1,566 @@ +//! Publisher-accepted transparent discovery metadata. Canonical wallet rows own value. +use crate::error::{EngineError, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use transparent_events::TransparentEvent; + +pub(crate) fn refused(code: &str) -> EngineError { + EngineError::Message(code.into()) +} + +pub use z_transparent_pir_v2 as acquisition; +pub use z_transparent_pir_v2::{ + Anchor as TransparentPirAnchor, Checkpoint as TransparentPirCheckpoint, + Record as TransparentPirRecord, Scope as TransparentPirScope, Script as TransparentPirScript, +}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectionState { + pub scope: Option, + #[serde(default)] + pub generation: u64, + #[serde(default)] + pub wallet_cursor: Option, + #[serde(default)] + pub bound_source: Option, + #[serde(default)] + pub rewind_floor: Option, + pub checkpoint_id: Option, + pub checkpoint_digest: Option, + #[serde(default)] + pub projection_invalidated: bool, + pub continuation: Option, + pub covered_through: u64, + pub complete: bool, + pub pending_pages: u64, + pub unresolved_spends: u64, + /// Identities and digests only. No second copy of wallet values or balance. + pub records: BTreeMap, + #[serde(default)] + pub projected_outputs: BTreeSet, + /// Web funding remains blocked independently of provisional-row deletion ownership. + /// None is legacy/unknown provenance and requires an explicit canonical rescan. + #[serde(default)] + pub unverified_outputs: Option>, + #[serde(default)] + pub projected_spends: BTreeSet, + #[serde(default)] + pub projected_transactions: BTreeSet, + #[serde(skip)] + pub durable: bool, +} + +pub(crate) fn hash(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +pub(crate) fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +pub(crate) fn parse(json: &str) -> Result { + if json.len() > 128 * 1024 * 1024 { + return Err(refused("transparent_pir_size_limit")); + } + serde_json::from_str(json).map_err(|_| refused("transparent_pir_input_invalid")) +} + +impl ProjectionState { + pub fn begin( + &mut self, + source: &str, + anchor: TransparentPirAnchor, + mut scripts: Vec, + ) -> Result { + scripts.sort_by(|a, b| a.script.cmp(&b.script)); + scripts.dedup(); + if !hash(source) || !hash(&anchor.hash) || scripts.is_empty() || scripts.len() > 4096 { + return Err(refused("transparent_pir_scope_invalid")); + } + if self + .bound_source + .as_deref() + .or_else(|| self.scope.as_ref().map(|s| s.source_id.as_str())) + .is_some_and(|old| old != source) + { + return Err(refused("transparent_pir_source_changed")); + } + if scripts.iter().any(|s| { + s.required_from > anchor.height + || crate::web::from_hex(&s.script) + .ok() + .is_none_or(|b| b.is_empty() || b.len() > 40) + }) { + return Err(refused("transparent_pir_scope_invalid")); + } + let scope_id = digest( + &serde_json::to_vec(&scripts).map_err(|_| refused("transparent_pir_scope_invalid"))?, + ); + if let Some(old) = &self.scope { + if old.source_id == source && old.anchor == anchor && old.scope_id == scope_id { + return Ok(old.clone()); + } + // A lower target cannot authenticate the saved anchor yet. Keep the + // private journal until canonical scanning reaches that height again. + if anchor.height < old.anchor.height { + return Err(refused("transparent_pir_rewind_required")); + } + } + let generation = self + .generation + .checked_add(1) + .ok_or_else(|| refused("transparent_pir_generation_exhausted"))?; + let same_source = self.bound_source.as_deref() == Some(source) + || self.scope.as_ref().is_some_and(|s| s.source_id == source); + if self + .scope + .as_ref() + .is_some_and(|old| old.scope_id != scope_id) + { + // Aggregate coverage belongs to this exact script inventory. The + // continuation retains each old script's independent coverage. + self.covered_through = 0; + } + self.bound_source = Some(source.into()); + self.generation = generation; + let scope = TransparentPirScope { + source_id: source.into(), + generation, + scope_id, + accepted_headers: vec![anchor.clone()], + anchor, + scripts, + }; + self.scope = Some(scope.clone()); + if !same_source { + self.checkpoint_id = None; + self.checkpoint_digest = None; + self.continuation = None; + self.covered_through = 0; + self.projected_outputs.clear(); + self.projected_spends.clear(); + self.projected_transactions.clear(); + } + self.complete = false; + self.durable = false; + Ok(scope) + } + + pub fn validate( + &self, + checkpoint: &TransparentPirCheckpoint, + ) -> Result, TransparentEvent)>> { + let scope = self + .scope + .as_ref() + .ok_or_else(|| refused("transparent_pir_consent_required"))?; + if checkpoint.source_id != scope.source_id + || checkpoint.generation != scope.generation + || checkpoint.scope_id != scope.scope_id + || checkpoint.anchor != scope.anchor + { + return Err(refused("transparent_pir_context_changed")); + } + let bytes = checkpoint_payload(checkpoint)?; + let replay = self.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id); + if replay { + if self.checkpoint_digest.as_ref() != Some(&digest(&bytes)) { + return Err(refused("transparent_pir_checkpoint_conflict")); + } + if !self.projection_invalidated { + return Ok(Vec::new()); + } + } + if !hash(&checkpoint.checkpoint_id) + || (!replay && checkpoint.previous_checkpoint_id != self.checkpoint_id) + || checkpoint.covered_through < self.covered_through + || checkpoint.covered_through > scope.anchor.height + || checkpoint.records.len() > 65_536 + || (checkpoint.complete + && (checkpoint.covered_through != scope.anchor.height + || checkpoint.pending_pages != 0 + || checkpoint.unresolved_spends != 0)) + { + return Err(refused("transparent_pir_checkpoint_invalid")); + } + if checkpoint.complete + && scope.scripts.iter().any(|s| { + !checkpoint.coverage.iter().any(|c| { + c.script == s.script + && c.required_from <= s.required_from + && c.through >= scope.anchor.height + && hash(&c.revision) + }) + }) + { + return Err(refused("transparent_pir_coverage_incomplete")); + } + let mut decoded = Vec::new(); + let mut identities = BTreeMap::new(); + let mut received_totals = BTreeMap::<[u8; 32], u64>::new(); + for record in &checkpoint.records { + if !hash(&record.revision) { + return Err(refused("transparent_pir_revision_invalid")); + } + let entry = scope + .scripts + .iter() + .find(|s| s.script == record.script) + .ok_or_else(|| refused("transparent_pir_script_not_owned"))?; + let script = crate::web::from_hex(&record.script) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + let bytes = crate::web::from_hex(&record.event) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + let event = TransparentEvent::from_bytes(&bytes) + .map_err(|_| refused("transparent_pir_record_invalid"))?; + if event.to_bytes() != bytes + || u64::from(event_height(event)) < entry.required_from + || u64::from(event_height(event)) > scope.anchor.height + { + return Err(refused("transparent_pir_record_invalid")); + } + if let TransparentEvent::Receive(receive) = event { + let total = received_totals.entry(receive.txid.0).or_default(); + *total = total + .checked_add(receive.value) + .ok_or_else(|| refused("transparent_pir_amount_invalid"))?; + zcash_protocol::value::Zatoshis::from_u64(*total) + .map_err(|_| refused("transparent_pir_amount_invalid"))?; + } + let key = event_key(event); + let fact = (event_height(event), digest(&bytes)); + if self.records.get(&key).is_some_and(|old| *old != fact) + || identities.insert(key, fact).is_some() + { + return Err(refused("transparent_pir_record_conflict")); + } + decoded.push((script, event)); + } + decoded.sort_by_key(|(_, e)| match e { + TransparentEvent::Receive(e) => (e.height, e.transaction_index, 0, e.output_index), + TransparentEvent::Spend(e) => (e.height, e.transaction_index, 1, e.input_index), + }); + Ok(decoded) + } + + pub fn prepare_rewind(&mut self, height: u32) { + self.records.retain(|_, (h, _)| *h <= height); + self.covered_through = self.covered_through.min(u64::from(height)); + self.complete = false; + } + + pub fn stage( + &mut self, + checkpoint: &TransparentPirCheckpoint, + records: &[(Vec, TransparentEvent)], + ) -> Result<()> { + for (_, event) in records { + self.records.insert( + event_key(*event), + (event_height(*event), digest(&event.to_bytes())), + ); + } + self.rewind_floor = None; + self.checkpoint_id = Some(checkpoint.checkpoint_id.clone()); + self.checkpoint_digest = Some(digest(&checkpoint_payload(checkpoint)?)); + let mut saved = checkpoint.clone(); + saved.records.clear(); + self.continuation = Some(saved); + self.covered_through = checkpoint.covered_through; + self.complete = checkpoint.complete; + self.pending_pages = checkpoint.pending_pages; + self.unresolved_spends = checkpoint.unresolved_spends; + self.durable = false; + Ok(()) + } + + pub fn acknowledge(&mut self, id: &str) -> Result<()> { + if self.checkpoint_id.as_deref() != Some(id) { + return Err(refused("transparent_pir_checkpoint_changed")); + } + self.durable = true; + Ok(()) + } + + pub fn invalidate_projection(&mut self) { + if self.bound_source.is_none() && self.scope.is_none() { + return; + } + // The scanner erased canonical rows, not acquisition progress. Keep the + // continuation and its owed cover queries; replay its records atomically. + self.projection_invalidated = true; + self.unverified_outputs = Some(BTreeSet::new()); + self.complete = false; + self.durable = false; + self.projected_outputs.clear(); + self.projected_spends.clear(); + self.projected_transactions.clear(); + } + + pub fn rewind(&mut self, height: u32) { + if self.bound_source.is_none() && self.scope.is_none() { + return; + } + self.rewind_floor = Some(self.rewind_floor.map_or(height, |old| old.min(height))); + self.projection_invalidated = true; + self.complete = false; + self.covered_through = self.covered_through.min(u64::from(height)); + // Canonical rows have their own rewind authority. Acquisition must + // retain its source evidence and owed cover queries for core revalidation. + self.durable = false; + } + + pub fn status(&self) -> serde_json::Value { + serde_json::json!({ "trust":"publisher-provided-confirmed-history", "spendingEnabled":false, + "completion":if self.complete && !self.projection_invalidated && self.durable { "complete-for-enrolled-scope" } else if self.complete && !self.projection_invalidated { "checkpoint-pending" } else { "incomplete" }, + "scope": self.scope.as_ref().map(|s|serde_json::json!({"sourceId":s.source_id,"scopeId":s.scope_id,"generation":s.generation,"anchor":s.anchor,"scriptCount":s.scripts.len()})), "checkpointId":self.checkpoint_id, "coveredThrough":self.covered_through, + "pendingPages":self.pending_pages, "unresolvedSpends":self.unresolved_spends }) + } +} + +pub(crate) fn event_height(event: TransparentEvent) -> u32 { + match event { + TransparentEvent::Receive(e) => e.height, + TransparentEvent::Spend(e) => e.height, + } +} +pub(crate) fn event_key(event: TransparentEvent) -> String { + match event { + TransparentEvent::Receive(e) => { + format!("r:{}:{}", crate::web::to_hex(&e.txid.0), e.output_index) + } + TransparentEvent::Spend(e) => format!( + "s:{}:{}:{}:{}", + crate::web::to_hex(&e.spending_txid.0), + e.input_index, + crate::web::to_hex(&e.spent_txid.0), + e.spent_output_index + ), + } +} + +fn checkpoint_payload(checkpoint: &TransparentPirCheckpoint) -> Result> { + let mut normalized = checkpoint.clone(); + normalized.previous_checkpoint_id = None; + serde_json::to_vec(&normalized).map_err(|_| refused("transparent_pir_input_invalid")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn inventory(count: usize) -> Vec { + (0..count) + .map(|index| TransparentPirScript { + script: format!("76a914{index:040x}88ac"), + required_from: 0, + }) + .collect() + } + + #[test] + fn scope_cap_counts_canonical_inventory_before_rejecting_duplicates() { + let unique = inventory(2049); + let mut duplicates = unique.clone(); + duplicates.extend(unique.clone()); + let anchor = TransparentPirAnchor { + height: 3, + hash: "03".repeat(32), + }; + let scope = ProjectionState::default() + .begin(&"ab".repeat(32), anchor.clone(), duplicates) + .unwrap(); + let expected = ProjectionState::default() + .begin(&"ab".repeat(32), anchor, unique) + .unwrap(); + assert_eq!(scope.scripts.len(), 2049); + assert_eq!(scope.scope_id, expected.scope_id); + } + + #[test] + fn scope_unique_cap_remains_4096_and_refusal_preserves_state() { + let anchor = TransparentPirAnchor { + height: 3, + hash: "03".repeat(32), + }; + assert_eq!( + ProjectionState::default() + .begin(&"ab".repeat(32), anchor.clone(), inventory(4096)) + .unwrap() + .scripts + .len(), + 4096 + ); + let mut state = enrolled(); + let before = serde_json::to_vec(&state).unwrap(); + assert!(state + .begin(&"ab".repeat(32), anchor, inventory(4097)) + .is_err()); + assert!( + serde_json::to_vec(&state).unwrap() == before, + "scope refusal must preserve private state" + ); + } + + #[test] + fn scope_canonicalization_does_not_merge_distinct_history_floors() { + let scripts = [0, 1, 1] + .into_iter() + .map(|required_from| TransparentPirScript { + script: "51".into(), + required_from, + }) + .collect(); + let scope = ProjectionState::default() + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 3, + hash: "03".repeat(32), + }, + scripts, + ) + .unwrap(); + assert_eq!(scope.scripts.len(), 2); + assert_eq!(scope.scripts[0].required_from, 0); + assert_eq!(scope.scripts[1].required_from, 1); + } + + fn enrolled() -> ProjectionState { + let mut state = ProjectionState::default(); + let scope = state + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 3, + hash: "03".repeat(32), + }, + vec![TransparentPirScript { + script: "51".into(), + required_from: 0, + }], + ) + .unwrap(); + let checkpoint = TransparentPirCheckpoint { + source_id: scope.source_id, + scope_id: scope.scope_id, + generation: scope.generation, + anchor: scope.anchor, + checkpoint_id: "cd".repeat(32), + previous_checkpoint_id: None, + covered_through: 3, + complete: false, + pending_pages: 1, + unresolved_spends: 0, + records: vec![], + coverage: vec![], + rewind_to: None, + continuation: "opaque-private-journal-and-cover-debt".into(), + }; + state.stage(&checkpoint, &[]).unwrap(); + state + } + #[test] + fn same_height_scope_change_preserves_private_checkpoint_for_core_revalidation() { + let mut state = enrolled(); + let saved = state.continuation.clone().unwrap(); + let scope = state + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 3, + hash: "83".repeat(32), + }, + state.scope.as_ref().unwrap().scripts.clone(), + ) + .unwrap(); + assert_eq!(scope.anchor.hash, "83".repeat(32)); + assert!( + serde_json::to_vec(&state.continuation).unwrap() + == serde_json::to_vec(&Some(saved)).unwrap(), + "fork enrollment must preserve private evidence and debt" + ); + assert!(!state.complete); + } + #[test] + fn canonical_rewind_preserves_private_journal_and_refuses_lower_enrollment() { + let mut state = enrolled(); + let saved = serde_json::to_vec(&state.continuation).unwrap(); + state.rewind(1); + assert!( + serde_json::to_vec(&state.continuation).unwrap() == saved, + "canonical rewind must retain private cover debt" + ); + assert_eq!(state.rewind_floor, Some(1)); + assert!(state + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 1, + hash: "01".repeat(32) + }, + state.scope.as_ref().unwrap().scripts.clone() + ) + .is_err()); + assert!( + serde_json::to_vec(&state.continuation).unwrap() == saved, + "closed lower target must preserve checkpoint" + ); + } + #[test] + fn expanded_scope_frontier_is_incomplete_without_erasing_acquisition() { + let mut state = enrolled(); + let saved = serde_json::to_vec(&state.continuation).unwrap(); + let mut scripts = state.scope.as_ref().unwrap().scripts.clone(); + scripts.push(TransparentPirScript { + script: "52".into(), + required_from: 0, + }); + let scope = state + .begin( + &"ab".repeat(32), + TransparentPirAnchor { + height: 3, + hash: "03".repeat(32), + }, + scripts, + ) + .unwrap(); + assert_eq!(state.covered_through, 0); + assert!( + serde_json::to_vec(&state.continuation).unwrap() == saved, + "scope growth must preserve acquisition and cover debt" + ); + let partial = TransparentPirCheckpoint { + source_id: scope.source_id, + scope_id: scope.scope_id, + generation: scope.generation, + anchor: scope.anchor, + checkpoint_id: "de".repeat(32), + previous_checkpoint_id: state.checkpoint_id.clone(), + covered_through: 0, + complete: false, + pending_pages: 1, + unresolved_spends: 0, + records: vec![], + coverage: vec![], + rewind_to: None, + continuation: "continued-private-journal-and-cover-debt".into(), + }; + state.validate(&partial).unwrap(); + } + + #[test] + fn never_enrolled_rewind_preserves_exact_serialized_state() { + let mut state = ProjectionState::default(); + let saved = serde_json::to_vec(&state).unwrap(); + state.rewind(1); + assert_eq!(serde_json::to_vec(&state).unwrap(), saved); + assert!(state.rewind_floor.is_none()); + } +} diff --git a/crates/z-engine/src/web/history.rs b/crates/z-engine/src/web/history.rs index b135785..aa603d4 100644 --- a/crates/z-engine/src/web/history.rs +++ b/crates/z-engine/src/web/history.rs @@ -14,6 +14,21 @@ pub fn entries( let mut rows: Vec = aggs .iter() .map(|(txid, agg)| { + #[cfg(feature = "transparent-pir")] + let (spent_zat, sent_note_count) = ( + agg.spent_zat.saturating_add(agg.pir_spent_zat), + agg.spent_notes.saturating_add(agg.pir_spent_notes), + ); + #[cfg(not(feature = "transparent-pir"))] + let (spent_zat, sent_note_count) = (agg.spent_zat, agg.spent_notes); + #[cfg(feature = "transparent-pir")] + let (received_zat, received_note_count) = ( + agg.received_zat + .saturating_add(agg.pir_received_zat.unwrap_or(0)), + agg.received_notes.saturating_add(agg.pir_received_notes), + ); + #[cfg(not(feature = "transparent-pir"))] + let (received_zat, received_note_count) = (agg.received_zat, agg.received_notes); // `received_zat` omits change, while per-pool receipts include it. // A wallet's balance delta must include change or a fee-only // self-transfer appears to lose its entire input note. @@ -30,18 +45,18 @@ pub fn entries( let account_delta_zat = if pool_received > 0 || pool_spent > 0 { pool_received as i64 - pool_spent as i64 } else { - agg.received_zat as i64 - agg.spent_zat as i64 + received_zat as i64 - spent_zat as i64 }; HistoryEntry { txid: WebWallet::display_txid(txid), mined_height: agg.mined_height, expiry_height: agg.expiry_height, account_delta_zat, - spent_zat: agg.spent_zat, - received_zat: agg.received_zat, + spent_zat, + received_zat, fee_zat: agg.fee_zat, - sent_note_count: agg.spent_notes, - received_note_count: agg.received_notes, + sent_note_count, + received_note_count, memo_count: agg.memos.len() as u32, has_change: agg.has_change, is_shielding: agg.is_shielding, diff --git a/crates/z-engine/src/web/mod.rs b/crates/z-engine/src/web/mod.rs index 253c090..59d96a0 100644 --- a/crates/z-engine/src/web/mod.rs +++ b/crates/z-engine/src/web/mod.rs @@ -84,6 +84,51 @@ pub fn from_hex(s: &str) -> Result, String> { Ok(out) } +/// Add explicit all-pool transaction positions to the synthetic funding fixtures. +/// These accepted-chain blocks exercise the public scanner; they do not prove consensus inclusion. +#[cfg(all(test, feature = "transparent-inputs"))] +pub(crate) fn scan_address_fixture_structure(w: &mut WebWallet) { + use zcash_client_backend::proto::compact_formats::{CompactBlock, CompactTx, TxOut}; + let blocks: Vec<_> = (w.transparent_next_height()..=w.scanned_height()) + .map(|height| { + let mut by_tx = std::collections::BTreeMap::>::new(); + for u in w.utxos.iter().filter(|u| u.height == height) { + let outputs = by_tx.entry(u.txid.clone()).or_default(); + outputs.resize(u.index as usize + 1, TxOut::default()); + outputs[u.index as usize] = TxOut { + value: u.value_zat, + script_pub_key: from_hex(&u.script).unwrap(), + }; + } + let mut transactions = vec![CompactTx { + index: 0, + txid: vec![0xff; 32], + ..Default::default() + }]; + transactions.extend(by_tx.into_iter().enumerate().map(|(index, (txid, vout))| { + CompactTx { + index: index as u64 + 1, + txid: from_hex(&txid).unwrap(), + vout, + ..Default::default() + } + })); + CompactBlock { + height: u64::from(height), + hash: from_hex(w.hash_at(height).unwrap()).unwrap(), + prev_hash: w + .hash_at(height.saturating_sub(1)) + .map(|s| from_hex(s).unwrap()) + .unwrap_or_else(|| vec![0; 32]), + vtx: transactions, + ..Default::default() + } + }) + .collect(); + w.apply_transparent_blocks(&encode_delimited(blocks)) + .unwrap(); +} + #[cfg(test)] mod tests { use super::*; @@ -312,7 +357,8 @@ mod tests { ) .unwrap(); assert_eq!(n, 1); - assert!(w.balance().transparent_available >= 100000); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 100000); assert!(w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).is_err()); } @@ -331,6 +377,7 @@ mod tests { r#"{"utxos":[{"txid":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ) .unwrap(); + scan_address_fixture_structure(&mut w); let raw = w .prove_shield(REGTEST_FAUCET_MNEMONIC, 1) .unwrap_or_else(|e| panic!("prove_shield: {e}")); diff --git a/crates/z-engine/src/web/public_data.rs b/crates/z-engine/src/web/public_data.rs index 4e12d76..29d8175 100644 --- a/crates/z-engine/src/web/public_data.rs +++ b/crates/z-engine/src/web/public_data.rs @@ -1,7 +1,7 @@ //! Wallet-independent block-range data. Coverage is separate from shielded scan //! progress so an old snapshot never claims it has scanned transparent activity. -use super::store::StoredUtxo; +use super::store::{CoinbaseStatus, StoredUtxo}; use super::{decode_delimited, to_hex, WebWallet}; use crate::error::{EngineError, Result}; use serde::{Deserialize, Serialize}; @@ -180,7 +180,7 @@ mod tests { scan(&mut w, vec![block(100)]); assert_eq!(w.balance().transparent_available, 80_000); let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); - assert!(restored.utxos[0].coinbase); + assert_eq!(restored.utxos[0].coinbase, CoinbaseStatus::Coinbase); assert_eq!(restored.balance().transparent_available, 80_000); } } @@ -379,6 +379,31 @@ impl WebWallet { )); } let scripts = self.public_transparent_scripts()?; + let mut receipts = Vec::new(); + let mut spends = Vec::new(); + for block in &blocks { + for tx in &block.vtx { + let id = to_hex(&tx.txid); + for (index, output) in tx.vout.iter().enumerate() { + if scripts + .iter() + .any(|(script, _)| *script == output.script_pub_key) + { + receipts.push((id.clone(), index as u32, output.value)); + } + } + for input in &tx.vin { + spends.push(( + to_hex(&input.prevout_txid), + input.prevout_index, + id.clone(), + Some(block.height as u32), + )); + } + } + } + self.preflight_transparent_receipts(&receipts)?; + self.preflight_transparent_spends(&spends)?; self.transparent_scan_required = true; let mut found = 0; for block in blocks { @@ -395,6 +420,16 @@ impl WebWallet { involved = true; } } + involved |= self + .record_transparent_receipt_spend( + &prev, + input.prevout_index, + &txid, + Some(height), + ) + .is_some(); + #[cfg(feature = "transparent-pir")] + self.record_public_transparent_spend(&txid, &prev, input.prevout_index); } for (index, output) in tx.vout.iter().enumerate() { let Some((_, address)) = scripts @@ -410,7 +445,7 @@ impl WebWallet { .find(|u| u.txid == txid && u.index == index as u32) { u.height = height; - u.coinbase = tx.index == 0; + u.coinbase = CoinbaseStatus::from_transaction(tx.index == 0); u.value_zat = output.value; u.script = to_hex(&output.script_pub_key); } else { @@ -421,12 +456,21 @@ impl WebWallet { value_zat: output.value, height, address: address.clone(), - coinbase: tx.index == 0, + coinbase: CoinbaseStatus::from_transaction(tx.index == 0), spent: false, spent_in: None, }); found += 1; } + self.credit_transparent_receipt( + &txid, + index as u32, + output.value, + Some(height), + false, + ); + #[cfg(feature = "transparent-pir")] + self.record_public_transparent_output(&txid, index as u32); } if involved { let agg = self.txs_mut().entry(txid).or_default(); @@ -442,12 +486,14 @@ impl WebWallet { hash: to_hex(&block.hash), }); } + #[cfg(feature = "transparent-pir")] + self.confirm_complete_public_transparent_outputs(); self.recompute_pool_fields(); Ok(found) } #[cfg(feature = "transparent-inputs")] - fn public_transparent_scripts(&mut self) -> Result, String)>> { + pub(crate) fn public_transparent_scripts(&mut self) -> Result, String)>> { use transparent::{ address::{Script, TransparentAddress}, keys::{NonHardenedChildIndex, TransparentKeyScope}, diff --git a/crates/z-engine/src/web/scan.rs b/crates/z-engine/src/web/scan.rs index cb1e8fb..0b011fc 100644 --- a/crates/z-engine/src/web/scan.rs +++ b/crates/z-engine/src/web/scan.rs @@ -665,6 +665,8 @@ impl WebWallet { } if hit == 1 { self.mark_spend_nf(&nf_hex, Some(spent_in)); + #[cfg(feature = "transparent-pir")] + self.transparent_pir.projected_transactions.remove(spent_in); let agg = self.txs_mut().entry(spent_in.to_string()).or_default(); agg.mined_height = Some(height); if newly { @@ -756,6 +758,10 @@ impl WebWallet { } fn push_note(&mut self, note: TrackedNote) { + #[cfg(feature = "transparent-pir")] + self.transparent_pir + .projected_transactions + .remove(¬e.txid); if self.notes().iter().any(|n| { n.txid == note.txid && n.output_index == note.output_index && n.pool == note.pool }) { @@ -765,11 +771,9 @@ impl WebWallet { } let agg = self.txs_mut().entry(note.txid.clone()).or_default(); // A send/shield we constructed already filled spent/received; only stamp height. - let from_wallet_construct = agg.spent_notes > 0 - || agg.is_shielding - || (agg.mined_height.is_none() && agg.received_notes > 0); + let preaccounted_receipt = agg.receipt_totals_recorded; agg.mined_height = Some(note.mined_height); - if !from_wallet_construct { + if !preaccounted_receipt { agg.received_zat = agg.received_zat.saturating_add(note.value_zat); agg.received_notes = agg.received_notes.saturating_add(1); agg.has_change = agg.has_change || note.is_change; @@ -795,6 +799,25 @@ impl WebWallet { if let Some(tb) = tx.transparent_bundle() { use zcash_keys::encoding::AddressCodec; let ours = self.transparent_address().map(str::to_string); + let mut receipts = Vec::new(); + for (index, output) in tb.vout.iter().enumerate() { + if let Some(address) = output.recipient_address() { + let encoded = address.encode(&self.network); + if ours.as_deref() == Some(encoded.as_str()) + || self.utxos.iter().any(|u| u.address == encoded) + { + receipts.push((txid.clone(), index as u32, u64::from(output.value()))); + } + } + } + self.preflight_transparent_receipts(&receipts)?; + if receipts.iter().any(|(id, index, value)| { + self.utxos + .iter() + .any(|u| u.txid == *id && u.index == *index && u.value_zat != *value) + }) { + return Err(EngineError::Message("transparent_receipt_conflict".into())); + } for (i, out) in tb.vout.iter().enumerate() { let Some(addr) = out.recipient_address() else { continue; @@ -806,12 +829,27 @@ impl WebWallet { continue; } let idx = i as u32; - if self.utxos.iter().any(|u| u.txid == txid && u.index == idx) { + if let Some(u) = self + .utxos + .iter_mut() + .find(|u| u.txid == txid && u.index == idx) + { + if u.coinbase != super::store::CoinbaseStatus::Coinbase { + u.coinbase = + super::store::CoinbaseStatus::from_transaction(tb.is_coinbase()); + } + self.credit_transparent_receipt( + &txid, + idx, + u64::from(out.value()), + Some(0), + false, + ); found += 1; continue; } self.utxos.push(super::store::StoredUtxo { - coinbase: false, + coinbase: super::store::CoinbaseStatus::from_transaction(tb.is_coinbase()), txid: txid.clone(), index: idx, script: super::to_hex(&out.script_pubkey().0 .0), @@ -821,6 +859,7 @@ impl WebWallet { spent: false, spent_in: None, }); + self.credit_transparent_receipt(&txid, idx, u64::from(out.value()), Some(0), false); found += 1; } } @@ -892,7 +931,14 @@ impl WebWallet { .map(BlockHeight::from_u32); let decrypted = decrypt_transaction(&self.network(), mined, Some(tip), tx, &keys); let mut memos = Vec::new(); - let mut inbound_zat = 0u64; + let known_receipts: std::collections::HashSet<_> = self + .notes() + .iter() + .filter(|note| note.txid == txid) + .map(|note| (note.pool.clone(), note.output_index)) + .collect(); + let mut unaccounted_zat = 0u64; + let mut unaccounted_n = 0u32; let mut inbound_n = 0u32; let mut sapling_received = 0u64; let mut orchard_received = 0u64; @@ -919,7 +965,14 @@ impl WebWallet { TransferType::Incoming | TransferType::AccountInternal ) { let value = o.note().0.value().inner(); - inbound_zat = inbound_zat.saturating_add(value); + let pool = match o.note().1 { + orchard::ValuePool::Orchard => "orchard", + orchard::ValuePool::Ironwood => "ironwood", + }; + if !known_receipts.contains(&(pool.to_string(), o.index() as u32)) { + unaccounted_zat = unaccounted_zat.saturating_add(value); + unaccounted_n += 1; + } let pool_received = match o.note().1 { orchard::ValuePool::Orchard => &mut orchard_received, orchard::ValuePool::Ironwood => &mut ironwood_received, @@ -943,8 +996,11 @@ impl WebWallet { TransferType::Incoming | TransferType::AccountInternal ) { let value = o.note().value().inner(); + if !known_receipts.contains(&("sapling".to_string(), o.index() as u32)) { + unaccounted_zat = unaccounted_zat.saturating_add(value); + unaccounted_n += 1; + } sapling_received = sapling_received.saturating_add(value); - inbound_zat = inbound_zat.saturating_add(value); inbound_n += 1; } else if matches!(o.transfer_type(), TransferType::Outgoing) { outgoing_shielded_zat = @@ -1014,9 +1070,10 @@ impl WebWallet { }; let expected_incoming = self.notes().iter().any(|note| note.txid == txid); let agg = self.txs_mut().entry(txid.clone()).or_default(); - if agg.mined_height.is_none() && inbound_n > 0 && agg.received_notes == 0 { - agg.received_zat = inbound_zat; - agg.received_notes = inbound_n; + if inbound_n > 0 && !agg.receipt_totals_recorded { + agg.received_zat = agg.received_zat.saturating_add(unaccounted_zat); + agg.received_notes = agg.received_notes.saturating_add(unaccounted_n); + agg.receipt_totals_recorded = true; agg.sapling_received = sapling_received; agg.orchard_received = orchard_received; agg.ironwood_received = ironwood_received; diff --git a/crates/z-engine/src/web/scan_tests.rs b/crates/z-engine/src/web/scan_tests.rs index 33483f8..5687421 100644 --- a/crates/z-engine/src/web/scan_tests.rs +++ b/crates/z-engine/src/web/scan_tests.rs @@ -241,3 +241,318 @@ fn serial_and_parallel_batches_reject_bad_shielded_data_and_chain_order() { assert_eq!(errors[0], errors[1], "{defect}"); } } + +#[test] +fn observed_spend_before_same_block_receipt_is_not_wallet_construction() { + for ironwood in [false, true] { + let mut w = wallet(REGTEST_FAUCET_MNEMONIC); + let (first, nf) = action( + &w, + 700, + { + let mut n = [0; 32]; + n[0] = 1; + n + }, + ironwood, + ); + let (second, _) = action(&w, 500, nf, ironwood); + let mut blocks = Vec::new(); + for (offset, a) in [first, second].into_iter().enumerate() { + let mut tx = CompactTx { + txid: vec![offset as u8 + 1; 32], + ..Default::default() + }; + if ironwood { + tx.ironwood_actions.push(a); + } else { + tx.actions.push(a); + } + blocks.push(CompactBlock { + height: u64::from(BIRTHDAY) + offset as u64, + hash: vec![offset as u8 + 1; 32], + prev_hash: vec![offset as u8; 32], + vtx: vec![tx], + ..Default::default() + }); + } + apply(&mut w, &blocks, 1); + w.recompute_pool_fields(); + let row = w + .history(20) + .into_iter() + .find(|r| r.txid == WebWallet::display_txid(&"02".repeat(32))) + .unwrap(); + assert_eq!((row.spent_zat, row.sent_note_count), (700, 1)); + assert_eq!( + (row.received_zat, row.received_note_count), + (500, 1), + "passive scan spend is not a constructed transaction" + ); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let row = cold + .history(20) + .into_iter() + .find(|r| r.txid == WebWallet::display_txid(&"02".repeat(32))) + .unwrap(); + assert_eq!((row.received_zat, row.received_note_count), (500, 1)); + } +} + +#[cfg(feature = "native")] +#[test] +fn old_wallet_write_pending_change_keeps_accounting_through_cold_confirmation() { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::{Note as WalletNote, Recipient}; + use zcash_primitives::transaction::Transaction; + use zcash_protocol::{ + consensus::{BlockHeight, BranchId}, + value::Zatoshis, + }; + let mut w = wallet(REGTEST_FAUCET_MNEMONIC); + let raw = crate::web::shielded_memo_transaction(&w, &w, "saved change", 30000, true); + let tx = Transaction::read(&raw[..], BranchId::Nu6_3).unwrap(); + let keys = std::collections::HashMap::from([(0u32, w.decode_ufvk().unwrap())]); + let decrypted = zcash_client_backend::decrypt_transaction( + &Network::Regtest, + None, + Some(BlockHeight::from_u32(BIRTHDAY)), + &tx, + &keys, + ); + let o = &decrypted.ironwood_outputs()[0]; + let outputs = [SentTransactionOutput::from_parts( + o.index(), + Recipient::InternalShielded { + receiving_account: 0, + external_address: None, + note: Box::new(WalletNote::Orchard { + note: o.note().0.clone(), + pool: o.note().1, + }), + }, + Zatoshis::from_u64(30000).unwrap(), + None, + )]; + let sent = SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + BIRTHDAY.into(), + 0, + &outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + w.store_transactions_to_be_sent(&[sent]).unwrap(); + let pending = w.pending_txs.clone(); + let id = to_hex(tx.txid().as_ref()); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("receiptTotalsRecorded"); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + assert_eq!(cold.pending_txs, pending); + assert_eq!( + cold.balance().ironwood_pending, + 30000, + "saved signed construction preserves its pool before confirmation" + ); + assert_eq!(cold.balance().orchard_pending, 0); + let block = cold.mined_raw_tx_block(&raw, 1).unwrap(); + cold.apply_compact_blocks_blob(&super::super::encode_delimited([block])) + .unwrap(); + cold.recompute_pool_fields(); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (30000, 1)); + let reopened = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + let row = reopened + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (30000, 1)); + assert_eq!(reopened.pending_txs, pending); +} + +#[cfg(all(feature = "native", feature = "transparent-pir"))] +#[test] +fn public_pir_spend_transfer_then_real_receipt_uses_independent_accounting() { + use crate::transparent_pir::acquisition::Coverage; + use crate::transparent_pir::{ + TransparentPirCheckpoint, TransparentPirRecord, TransparentPirScope, + }; + use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; + use zcash_client_backend::proto::compact_formats::CompactTxIn; + use zcash_primitives::transaction::{Authorized, Transaction, TransactionData}; + use zcash_protocol::consensus::{BlockHeight, BranchId}; + let mut decryptor = wallet(REGTEST_FAUCET_MNEMONIC); + let raw = crate::web::shielded_memo_transaction( + &decryptor, + &decryptor, + "independent receipt", + 30000, + true, + ); + let template = Transaction::read(&raw[..], BranchId::Nu6_3).unwrap(); + // One parsed transaction supplies both the real encrypted action and its + // actual public input. Proof/signature bytes are inert; no consensus claim. + let tx = TransactionData::::from_parts_v6( + BranchId::Nu6_3, + 0, + BlockHeight::from_u32(BIRTHDAY + 20), + Some(transparent::bundle::Bundle { + vin: vec![transparent::bundle::TxIn::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::address::Script::default(), + u32::MAX, + )], + vout: vec![], + authorization: transparent::bundle::Authorized, + }), + None, + None, + template.ironwood_bundle().cloned(), + ) + .freeze() + .unwrap(); + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + let id = to_hex(tx.txid().as_ref()); + let mut block = decryptor.mined_raw_tx_block(&raw, 1).unwrap(); + // The scanner independently trial-decrypts the real note before its later + // ingestion boundary is exercised in the target wallet. + decryptor + .apply_compact_blocks_blob(&super::super::encode_delimited([block.clone()])) + .unwrap(); + decryptor.recompute_pool_fields(); + let note = decryptor + .notes() + .iter() + .find(|n| n.txid == id) + .unwrap() + .clone(); + assert_eq!(note.value_zat, 30000); + block.vtx[0].vin = vec![CompactTxIn { + prevout_txid: vec![1; 32], + prevout_index: 0, + }]; + let mut w = wallet(REGTEST_FAUCET_MNEMONIC); + w.set_scanned( + BIRTHDAY + 2, + WebWallet::make_prior( + BIRTHDAY + 2, + zcash_primitives::block::BlockHash([3; 32]), + 0, + 0, + 0, + ), + ) + .unwrap(); + for (offset, byte) in [(0, 1), (1, 2), (2, 3)] { + w.block_hashes + .insert(BIRTHDAY + offset, format!("{byte:02x}").repeat(32)); + } + w.block_hashes.insert(BIRTHDAY - 1, "00".repeat(32)); + let scope: TransparentPirScope = + serde_json::from_str(&w.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let script = to_hex(&w.public_transparent_scripts().unwrap()[0].0); + let events = [ + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: BIRTHDAY - 1, + txid: Txid([1; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }), + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: BIRTHDAY, + spending_txid: Txid(*tx.txid().as_ref()), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([1; 32]), + spent_output_index: 0, + }), + ]; + let cp = TransparentPirCheckpoint { + source_id: scope.source_id, + generation: scope.generation, + scope_id: scope.scope_id, + anchor: scope.anchor, + checkpoint_id: "cd".repeat(32), + previous_checkpoint_id: None, + covered_through: u64::from(BIRTHDAY + 2), + complete: true, + pending_pages: 0, + unresolved_spends: 0, + records: events + .into_iter() + .map(|e| TransparentPirRecord { + script: script.clone(), + event: to_hex(&e.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }) + .collect(), + coverage: scope + .scripts + .into_iter() + .map(|s| Coverage { + script: s.script, + required_from: s.required_from, + through: u64::from(BIRTHDAY + 2), + revision: "ef".repeat(32), + }) + .collect(), + rewind_to: None, + continuation: String::new(), + }; + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&cp).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&cp.checkpoint_id) + .unwrap(); + let mut blocks: Vec<_> = (0..3) + .map(|i| CompactBlock { + height: u64::from(BIRTHDAY + i), + hash: vec![i as u8 + 1; 32], + prev_hash: vec![i as u8; 32], + vtx: vec![CompactTx { + index: 0, + txid: vec![100 + i as u8; 32], + ..Default::default() + }], + ..Default::default() + }) + .collect(); + blocks[0].vtx.push(block.vtx.remove(0)); + w.apply_transparent_blocks(&super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!( + (w.txs()[&id].spent_zat, w.txs()[&id].spent_notes), + (150000, 1) + ); + assert_eq!( + (w.txs()[&id].pir_spent_zat, w.txs()[&id].pir_spent_notes), + (0, 0) + ); + assert!(w.pending_txs.is_empty()); + // This is an ingestion-boundary test, not a claim that normal scheduling + // scans transparent blocks before shielded blocks at a new height. The + // public range above is checked against the target's accepted saved headers. + w.push_note(note.clone()); + w.recompute_pool_fields(); + w.push_note(note); + let assert_history = |wallet: &WebWallet| { + let row = wallet + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (30000, 1)); + assert_eq!((row.spent_zat, row.sent_note_count), (150000, 1)); + assert_eq!(row.account_delta_zat, -120000); + }; + assert_history(&w); + assert_history(&WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap()); +} diff --git a/crates/z-engine/src/web/store.rs b/crates/z-engine/src/web/store.rs index 8f8494f..118d2df 100644 --- a/crates/z-engine/src/web/store.rs +++ b/crates/z-engine/src/web/store.rs @@ -1,4 +1,7 @@ //! Versioned in-memory wallet snapshot (IndexedDB blob on the JS side). +#[cfg(feature = "transparent-pir")] +#[path = "transparent_pir.rs"] +mod transparent_pir_impl; use super::history; use super::live_trees::{self, StoredLiveTrees}; @@ -175,11 +178,55 @@ fn root_range(roots: &[StoredSubtreeRoot]) -> (u64, u64) { (from as u64, roots.len() as u64) } +/// Coinbase maturity can only be waived by independently observed transaction structure. +/// Old boolean false snapshots and address-UTXO replies did not carry that evidence. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum CoinbaseStatus { + #[default] + Unknown, + Coinbase, + NonCoinbase, +} + +impl CoinbaseStatus { + pub(crate) fn from_transaction(coinbase: bool) -> Self { + if coinbase { + Self::Coinbase + } else { + Self::NonCoinbase + } + } + + pub(crate) fn from_unverified(coinbase: bool) -> Self { + if coinbase { + Self::Coinbase + } else { + Self::Unknown + } + } + + fn from_snapshot<'de, D: serde::Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Saved { + Current(CoinbaseStatus), + Legacy(bool), + } + Ok(match Saved::deserialize(deserializer)? { + Saved::Current(status) => status, + Saved::Legacy(value) => Self::from_unverified(value), + }) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub(crate) struct StoredUtxo { - #[serde(default)] - pub coinbase: bool, + #[serde(default, deserialize_with = "CoinbaseStatus::from_snapshot")] + pub coinbase: CoinbaseStatus, pub txid: String, pub index: u32, pub script: String, @@ -191,6 +238,72 @@ pub(crate) struct StoredUtxo { pub spent_in: Option, } +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TransparentReceipt { + pub value_zat: u64, + #[serde(default)] + pub builder: bool, + #[serde(default)] + pub observed_height: Option, + #[serde(default)] + pub spent_in: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct TransparentReceiptSpend { + pub txid: String, + pub height: Option, + #[serde(default)] + pub accounted: bool, +} + +fn legacy_receipt_accounting() -> bool { + true +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct ShieldedInputOrigins(pub(crate) Option>); + +impl Default for ShieldedInputOrigins { + fn default() -> Self { + Self(Some(BTreeSet::new())) + } +} + +fn unknown_shielded_input_origins() -> ShieldedInputOrigins { + ShieldedInputOrigins(None) +} + +fn read_transparent_receipts<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum Saved { + Receipt(TransparentReceipt), + Legacy(u64), + } + Ok(BTreeMap::::deserialize(deserializer)? + .into_iter() + .map(|(index, saved)| { + let receipt = match saved { + Saved::Receipt(receipt) => receipt, + Saved::Legacy(value_zat) => TransparentReceipt { + value_zat, + ..Default::default() + }, + }; + (index, receipt) + }) + .collect()) +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TxAgg { @@ -199,6 +312,39 @@ pub struct TxAgg { pub spent_zat: u64, pub received_notes: u32, pub spent_notes: u32, + /// A complete WalletWrite or full-transaction receipt producer filled the + /// generic receipt totals. Passive spends are not receipt accounting. + #[serde(default)] + pub(crate) receipt_totals_recorded: bool, + /// Exact transparent receipts already credited by an independent source. + /// This is accounting ownership, not inclusion, deletion or funding authority. + #[serde(default, deserialize_with = "read_transparent_receipts")] + pub(crate) accounted_transparent_receipts: BTreeMap, + // Fresh aggregates default to current accounting; absent serialized markers + // identify old aggregates whose generic transparent share is ambiguous. + #[serde(default = "legacy_receipt_accounting")] + pub(crate) receipt_accounting_legacy: bool, + /// Exact independent transparent debit credits owned by this spender. + #[serde(default)] + pub(crate) accounted_transparent_spends: BTreeMap>, + /// Exact owned shielded receiving transactions consumed by our saved intent. + /// This survives origin-note truncation and grants no funding authority. + #[serde(default = "unknown_shielded_input_origins")] + pub(crate) shielded_input_origins: ShieldedInputOrigins, + #[cfg(feature = "transparent-pir")] + #[serde(default)] + pub(crate) pir_received_zat: Option, + #[cfg(feature = "transparent-pir")] + #[serde(default)] + pub(crate) pir_received_notes: u32, + // The share of generic history totals contributed by PIR-owned spend marks. + // Public observation transfers this accounting without removing its value. + #[cfg(feature = "transparent-pir")] + #[serde(default)] + pub(crate) pir_spent_zat: u64, + #[cfg(feature = "transparent-pir")] + #[serde(default)] + pub(crate) pir_spent_notes: u32, pub has_change: bool, #[serde(default)] pub is_shielding: bool, @@ -256,6 +402,9 @@ pub struct TxAgg { impl TxAgg { pub(crate) fn has_shielded_activity(&self) -> bool { self.is_shielding + || self.outgoing_shielded_zat > 0 + || !self.memos.is_empty() + || self.memo_recovered == Some(true) || self.sapling_received > 0 || self.sapling_spent > 0 || self.orchard_received > 0 @@ -345,6 +494,9 @@ struct Snapshot { transparent_scan: Option, #[serde(default)] transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + #[serde(default)] + transparent_pir: crate::transparent_pir::ProjectionState, #[serde(default)] memo_scan: Option, #[serde(default)] @@ -420,6 +572,8 @@ struct SnapshotRef<'a> { utxos: &'a [StoredUtxo], transparent_scan: &'a Option, transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + transparent_pir: &'a crate::transparent_pir::ProjectionState, memo_scan: &'a Option, block_hashes: &'a BTreeMap, view_only: bool, @@ -524,6 +678,8 @@ pub struct WebWallet { pub(crate) utxos: Vec, pub(crate) transparent_scan: Option, pub(crate) transparent_scan_required: bool, + #[cfg(feature = "transparent-pir")] + pub(crate) transparent_pir: crate::transparent_pir::ProjectionState, pub(crate) memo_scan: Option, pub(crate) block_hashes: BTreeMap, view_only: bool, @@ -601,6 +757,8 @@ impl WebWallet { utxos: Vec::new(), transparent_scan: None, transparent_scan_required: false, + #[cfg(feature = "transparent-pir")] + transparent_pir: Default::default(), memo_scan: None, block_hashes: BTreeMap::new(), view_only: false, @@ -767,17 +925,19 @@ impl WebWallet { let skip_pending: std::collections::HashSet = self .txs .iter() - .filter(|(_, a)| a.mined_height.is_none() && (a.spent_notes > 0 || a.is_shielding)) + .filter(|(_, a)| a.mined_height.is_none() && a.receipt_totals_recorded) .map(|(k, _)| k.clone()) .collect(); let note_txids: std::collections::HashSet<_> = self.notes.iter().map(|note| note.txid.as_str()).collect(); for (txid, agg) in self.txs.iter_mut() { + // Transparent inputs/receipts are rebuilt once even for a pending + // construction; shielded change below can still await compact rows. + agg.transparent_received = 0; + agg.transparent_spent = 0; if skip_pending.contains(txid) { continue; } - agg.transparent_received = 0; - agg.transparent_spent = 0; // A decrypted mempool receipt has no compact-note row yet. Its // pool totals are known from the full transaction and must survive // refresh/reload until mined notes become the source of truth. @@ -839,28 +999,43 @@ impl WebWallet { } } let utxos = self.utxos.clone(); + #[cfg(feature = "transparent-pir")] + let mut pir_receipts = BTreeMap::<&str, u64>::new(); + #[cfg(feature = "transparent-pir")] + for u in &utxos { + if self.pir_receipt_unaccounted(&u.txid, u.index) { + let value = pir_receipts.entry(u.txid.as_str()).or_default(); + *value = value.saturating_add(u.value_zat); + } + } for u in &utxos { let agg = self.txs.entry(u.txid.clone()).or_default(); if agg.mined_height.is_none() && u.height > 0 { agg.mined_height = Some(u.height); } bump_recv(agg, "transparent", u.value_zat); - agg.received_zat = agg.received_zat.max( - agg.transparent_received - .saturating_add(agg.sapling_received) - .saturating_add(agg.orchard_received) - .saturating_add(agg.ironwood_received), - ); - if agg.received_notes == 0 { + let received = agg + .transparent_received + .saturating_add(agg.sapling_received) + .saturating_add(agg.orchard_received) + .saturating_add(agg.ironwood_received); + #[cfg(feature = "transparent-pir")] + let received = + received.saturating_sub(pir_receipts.get(u.txid.as_str()).copied().unwrap_or(0)); + agg.received_zat = agg.received_zat.max(received); + #[cfg(feature = "transparent-pir")] + let owned = self + .transparent_pir + .projected_outputs + .contains(&format!("r:{}:{}", u.txid, u.index)) + && !agg.accounted_transparent_receipts.contains_key(&u.index); + #[cfg(not(feature = "transparent-pir"))] + let owned = false; + if !owned && agg.received_notes == 0 { agg.received_notes = 1; } if u.spent { if let Some(spent_in) = u.spent_in.as_ref() { - // A pending spender keeps its totals from construction; - // adding again on every recompute grew its "sent" amount. - if skip_pending.contains(spent_in) { - continue; - } let spent_txid = spent_in.clone(); let v = u.value_zat; let s = self.txs.entry(spent_txid).or_default(); @@ -870,6 +1045,40 @@ impl WebWallet { } } } + // Independently credited receipt history is not an unspent-output list. + // An omitted address refresh can remove funding without erasing which + // pool received that earlier output. These facts create no funding rows. + let represented: std::collections::HashSet<_> = + utxos.iter().map(|u| (u.txid.as_str(), u.index)).collect(); + for (id, agg) in &mut self.txs { + for (index, receipt) in &agg.accounted_transparent_receipts { + if !represented.contains(&(id.as_str(), *index)) { + agg.transparent_received = + agg.transparent_received.saturating_add(receipt.value_zat); + } + } + } + let represented_spends: std::collections::HashSet<_> = utxos + .iter() + .filter_map(|u| { + u.spent_in + .as_ref() + .filter(|_| u.spent) + .map(|spender| (spender.as_str(), u.txid.as_str(), u.index)) + }) + .collect(); + for (spender, agg) in &mut self.txs { + for (previous, outputs) in &agg.accounted_transparent_spends { + for (index, value) in outputs { + if !represented_spends.contains(&(spender.as_str(), previous.as_str(), *index)) + { + agg.transparent_spent = agg.transparent_spent.saturating_add(*value); + } + } + } + } + #[cfg(feature = "transparent-pir")] + self.reconcile_transparent_pir_history(); } pub fn record_recipient(&mut self, to: &str) { @@ -995,6 +1204,8 @@ impl WebWallet { utxos: &self.utxos, transparent_scan: &self.transparent_scan, transparent_scan_required: self.transparent_scan_required, + #[cfg(feature = "transparent-pir")] + transparent_pir: &self.transparent_pir, memo_scan: &self.memo_scan, block_hashes: &self.block_hashes, view_only: self.view_only, @@ -1072,6 +1283,8 @@ impl WebWallet { utxos: snap.utxos, transparent_scan: snap.transparent_scan, transparent_scan_required: snap.transparent_scan_required, + #[cfg(feature = "transparent-pir")] + transparent_pir: snap.transparent_pir, memo_scan: snap.memo_scan, block_hashes: snap.block_hashes, view_only: snap.view_only, @@ -1147,6 +1360,41 @@ impl WebWallet { } w.assert_consistent_leaf_hashes()?; w.repair_tree_cursors(); + // Older pending receipts lack an explicit accounting marker. Restore it + // only from a completed full-transaction receipt or exact retained saved + // transaction bytes, never from a generic spend/receipt count alone. + let note_txids: std::collections::BTreeSet<_> = + w.notes.iter().map(|n| n.txid.as_str()).collect(); + for (id, agg) in &mut w.txs { + if agg.mined_height.is_none() + && agg.history_metadata_complete + && agg.enhancement_complete + && agg.memo_recovered == Some(true) + && agg.received_notes > 0 + && !note_txids.contains(id.as_str()) + { + agg.receipt_totals_recorded = true; + } + } + let branch = zcash_protocol::consensus::BranchId::for_height( + &w.network, + BlockHeight::from_u32(w.scanned_height.saturating_add(1)), + ); + for (id, raw) in &w.pending_txs { + let Some(agg) = w + .txs + .get_mut(&to_hex(id.as_ref())) + .filter(|agg| !agg.receipt_totals_recorded && agg.received_notes > 0) + else { + continue; + }; + let mut bytes = raw.as_slice(); + if Transaction::read(&mut bytes, branch) + .is_ok_and(|tx| tx.txid() == *id && bytes.is_empty()) + { + agg.receipt_totals_recorded = true; + } + } w.reconcile_mined_spends(); w.prune_settled_shards(); w.next_note_id = w @@ -1163,6 +1411,10 @@ impl WebWallet { if let Some(live) = live_trees { w.restore_live_trees(&live); } + #[cfg(feature = "transparent-pir")] + { + w.transparent_pir.durable = w.transparent_pir.checkpoint_id.is_some(); + } w.recompute_pool_fields(); Ok(w) } @@ -1286,6 +1538,429 @@ impl WebWallet { self.balance_for_policy(&crate::confirmations_policy(self.network)) } + pub(crate) fn legacy_transparent_receipt_accounting(&self) -> bool { + self.pending_txs.keys().any(|id| { + self.txs + .get(&to_hex(id.as_ref())) + .is_some_and(|agg| agg.shielded_input_origins.0.is_none()) + }) || self.txs.values().any(|agg| { + agg.accounted_transparent_receipts.values().any(|receipt| { + receipt + .spent_in + .as_ref() + .is_some_and(|spent| !spent.accounted) + }) || agg.receipt_accounting_legacy + && (agg.transparent_received > 0 + || !agg.accounted_transparent_receipts.is_empty() + || agg.received_zat + > agg + .sapling_received + .saturating_add(agg.orchard_received) + .saturating_add(agg.ironwood_received)) + }) + } + + pub(crate) fn preflight_transparent_receipts( + &self, + receipts: &[(String, u32, u64)], + ) -> Result<()> { + if receipts.is_empty() { + return Ok(()); + } + self.ensure_current_transparent_receipts()?; + let mut seen = std::collections::HashMap::new(); + for (txid, index, value) in receipts { + if let Some(agg) = self.txs.get(txid) { + if agg.receipt_accounting_legacy + && (agg.received_zat > 0 || !agg.accounted_transparent_receipts.is_empty()) + { + return Err(EngineError::Message( + "transparent_pir_rescan_required".into(), + )); + } + if agg + .accounted_transparent_receipts + .get(index) + .is_some_and(|receipt| receipt.value_zat != *value) + { + return Err(EngineError::Message("transparent_receipt_conflict".into())); + } + } + if seen + .insert((txid.as_str(), *index), *value) + .is_some_and(|prior| prior != *value) + { + return Err(EngineError::Message("transparent_receipt_conflict".into())); + } + } + Ok(()) + } + + pub(crate) fn ensure_current_transparent_receipts(&self) -> Result<()> { + if self.legacy_transparent_receipt_accounting() { + return Err(EngineError::Message( + "transparent_pir_rescan_required".into(), + )); + } + #[cfg(feature = "transparent-pir")] + if self.transparent_pir_rescan_required() { + return Err(EngineError::Message( + "transparent_pir_rescan_required".into(), + )); + } + Ok(()) + } + + pub(crate) fn credit_transparent_receipt( + &mut self, + txid: &str, + index: u32, + value: u64, + observed_height: Option, + builder: bool, + ) { + // All producers preflight their complete batch before any wallet writes. + let spent_in = self + .utxos + .iter() + .find(|u| u.txid == txid && u.index == index && u.spent) + .and_then(|u| u.spent_in.clone()) + .filter(|spender| { + #[cfg(feature = "transparent-pir")] + { + let prefix = format!("s:{spender}:"); + let suffix = format!(":{txid}:{index}"); + return !self + .transparent_pir + .projected_spends + .iter() + .any(|claim| claim.starts_with(&prefix) && claim.ends_with(&suffix)); + } + #[cfg(not(feature = "transparent-pir"))] + { + let _ = spender; + true + } + }) + .map(|txid| TransparentReceiptSpend { + height: self.txs.get(&txid).and_then(|agg| agg.mined_height), + txid, + accounted: true, + }); + if let Some(spent) = &spent_in { + self.txs + .entry(spent.txid.clone()) + .or_default() + .accounted_transparent_spends + .entry(txid.to_owned()) + .or_default() + .insert(index, value); + } + let agg = self.txs.entry(txid.to_owned()).or_default(); + agg.receipt_accounting_legacy = false; + let receipt = agg + .accounted_transparent_receipts + .entry(index) + .or_insert_with(|| { + agg.received_zat = agg.received_zat.saturating_add(value); + agg.received_notes = agg.received_notes.saturating_add(1); + TransparentReceipt { + value_zat: value, + ..Default::default() + } + }); + receipt.builder |= builder; + if let Some(height) = observed_height { + receipt.observed_height = Some(receipt.observed_height.unwrap_or(0).max(height)); + } + if let Some(spent) = spent_in { + if receipt.spent_in.is_none() || spent.height.is_some() { + receipt.spent_in = Some(spent); + } + } + } + + pub(crate) fn preflight_transparent_spends( + &self, + spends: &[(String, u32, String, Option)], + ) -> Result<()> { + let mut seen = std::collections::HashMap::new(); + let mut checked_current = false; + for (previous, index, spender, height) in spends { + let receipt = self + .txs + .get(previous) + .and_then(|agg| agg.accounted_transparent_receipts.get(index)); + if receipt.is_none() + && !self + .utxos + .iter() + .any(|u| u.txid == *previous && u.index == *index) + { + continue; + } + if !checked_current { + self.ensure_current_transparent_receipts()?; + checked_current = true; + } + let prior = seen.entry((previous.as_str(), *index)).or_insert_with(|| { + receipt.and_then(|receipt| { + receipt + .spent_in + .as_ref() + .map(|spent| (spent.txid.as_str(), spent.height)) + }) + }); + if prior.is_some_and(|(prior, old_height)| { + prior != spender + && (height.is_none() || old_height.is_some_and(|height| height > 0)) + }) { + return Err(EngineError::Message("transparent_receipt_conflict".into())); + } + if prior.is_none_or(|(_, old_height)| height.is_some() || old_height.is_none()) { + *prior = Some((spender.as_str(), *height)); + } + } + Ok(()) + } + + pub(crate) fn credit_transparent_debit( + &mut self, + spender: &str, + previous: &str, + index: u32, + value: u64, + ) { + #[cfg(feature = "transparent-pir")] + self.release_pir_spend_ownership(spender, previous, index); + let agg = self.txs.entry(spender.to_owned()).or_default(); + if let std::collections::btree_map::Entry::Vacant(entry) = agg + .accounted_transparent_spends + .entry(previous.to_owned()) + .or_default() + .entry(index) + { + entry.insert(value); + agg.spent_zat = agg.spent_zat.saturating_add(value); + agg.spent_notes = agg.spent_notes.saturating_add(1); + } + } + + pub(crate) fn record_transparent_receipt_spend( + &mut self, + previous: &str, + index: u32, + spender: &str, + height: Option, + ) -> Option { + let receipt = self + .txs + .get_mut(previous)? + .accounted_transparent_receipts + .get_mut(&index)?; + let value = receipt.value_zat; + // The complete operation was preflighted before any wallet writes. + if height.is_some() + || receipt + .spent_in + .as_ref() + .is_none_or(|spent| spent.height.is_none_or(|height| height == 0)) + { + receipt.spent_in = Some(TransparentReceiptSpend { + txid: spender.to_owned(), + height, + accounted: true, + }); + } + let included_height = receipt + .spent_in + .as_ref() + .and_then(|spent| spent.height) + .filter(|height| *height > 0); + self.credit_transparent_debit(spender, previous, index, value); + if let Some(height) = included_height { + self.txs.entry(spender.to_owned()).or_default().mined_height = Some(height); + } + Some(value) + } + + pub(crate) fn ineligible_pending_transactions(&self) -> std::collections::HashSet { + // Debit keys preserve intent history. Only current independently mined + // input ownership can disqualify that intent from submission or change. + let mut included = std::collections::HashMap::new(); + for (previous, agg) in &self.txs { + for (index, receipt) in &agg.accounted_transparent_receipts { + if let Some(spent) = receipt + .spent_in + .as_ref() + .filter(|spent| spent.height.is_some_and(|height| height > 0)) + { + included.insert((previous.as_str(), *index), spent.txid.as_str()); + } + } + } + for output in &self.utxos { + let Some(spender) = output.spent_in.as_ref().filter(|spender| { + output.spent + && self + .txs + .get(*spender) + .is_some_and(|agg| agg.mined_height.is_some_and(|height| height > 0)) + }) else { + continue; + }; + #[cfg(feature = "transparent-pir")] + { + let prefix = format!("s:{spender}:"); + let suffix = format!(":{}:{}", output.txid, output.index); + if self + .transparent_pir + .projected_spends + .range(prefix.clone()..) + .take_while(|claim| claim.starts_with(&prefix)) + .any(|claim| claim.ends_with(&suffix)) + { + continue; + } + } + included + .entry((output.txid.as_str(), output.index)) + .or_insert(spender.as_str()); + } + let mut conflicts: std::collections::HashSet<_> = self + .txs + .iter() + .filter(|(id, agg)| { + agg.mined_height.is_none() + && agg + .accounted_transparent_spends + .iter() + .any(|(previous, outputs)| { + outputs.keys().any(|index| { + included + .get(&(previous.as_str(), *index)) + .is_some_and(|spender| *spender != id.as_str()) + }) + }) + }) + .map(|(id, _)| id.clone()) + .collect(); + let mut children = std::collections::HashMap::<&str, Vec<&str>>::new(); + for id in self.pending_txs.keys() { + let id = to_hex(id.as_ref()); + if self.txs.get(&id).is_some_and(|agg| { + agg.mined_height.is_none() && agg.shielded_input_origins.0.is_none() + }) { + conflicts.insert(id); + } + } + for (child, agg) in &self.txs { + if agg.mined_height.is_none() { + for parent in agg.accounted_transparent_spends.keys() { + children + .entry(parent.as_str()) + .or_default() + .push(child.as_str()); + } + if let Some(parents) = &agg.shielded_input_origins.0 { + for parent in parents { + children + .entry(parent.as_str()) + .or_default() + .push(child.as_str()); + } + } + } + } + for (parent, child) in self + .notes + .iter() + .filter_map(|note| { + note.spent_in + .as_deref() + .map(|child| (note.txid.as_str(), child)) + }) + .chain(self.spend_notes.iter().filter_map(|note| { + note.spent_in + .as_deref() + .map(|child| (note.txid.as_str(), child)) + })) + { + if self + .txs + .get(child) + .is_some_and(|agg| agg.mined_height.is_none()) + { + children.entry(parent).or_default().push(child); + } + } + let mut queue: std::collections::VecDeque<_> = conflicts.iter().cloned().collect(); + while let Some(parent) = queue.pop_front() { + for child in children.get(parent.as_str()).into_iter().flatten() { + if conflicts.insert((*child).to_owned()) { + queue.push_back((*child).to_owned()); + } + } + } + conflicts + } + + fn restore_pending_transparent_claims(&mut self) { + // Retained signed intents keep their exact known input reservations + // even when submission or pending change is ineligible. Restore only + // existing claims, never funding rows or values. + let claims: Vec<_> = self + .retained_signed_txs() + .filter_map(|(id, _)| { + self.txs + .get(&id) + .filter(|agg| agg.mined_height.is_none()) + .map(|agg| (id, agg.accounted_transparent_spends.clone())) + }) + .collect(); + for (spender, inputs) in claims { + for (previous, outputs) in inputs { + for index in outputs.keys() { + let receipt = self + .txs + .get(&previous) + .and_then(|agg| agg.accounted_transparent_receipts.get(index)); + if receipt + .and_then(|receipt| receipt.spent_in.as_ref()) + .is_some_and(|spent| spent.txid != spender) + || self.utxos.iter().any(|output| { + output.txid == previous + && output.index == *index + && output.spent + && output.spent_in.as_deref() != Some(spender.as_str()) + }) + { + continue; + } + if let Some(receipt) = self + .txs + .get_mut(&previous) + .and_then(|agg| agg.accounted_transparent_receipts.get_mut(index)) + { + receipt + .spent_in + .get_or_insert_with(|| TransparentReceiptSpend { + txid: spender.clone(), + height: None, + accounted: true, + }); + } + for output in &mut self.utxos { + if output.txid == previous && output.index == *index { + output.spent = true; + output.spent_in = Some(spender.clone()); + } + } + } + } + } + } + /// Current unspent balance under an explicit minimum for both change and received notes. pub fn balance_with_confirmations(&self, confirmations: u32) -> Result { Ok(self.balance_for_policy(&super::explicit_confirmations_policy(confirmations)?)) @@ -1305,6 +1980,7 @@ impl WebWallet { let mut orchard_p = 0u64; let mut ironwood_p = 0u64; let mut t_p = 0u64; + let conflicts = self.ineligible_pending_transactions(); let add = |avail: &mut u64, pend: &mut u64, v: u64, ready: bool| { if ready { *avail = avail.saturating_add(v); @@ -1313,18 +1989,18 @@ impl WebWallet { } }; for u in &self.utxos { - if u.spent { + if u.spent || conflicts.contains(&u.txid) { continue; } add( &mut t, &mut t_p, u.value_zat, - self.utxo_ready(u, target, policy), + self.utxo_ready(u, target, policy, &conflicts), ); } for n in &self.notes { - if n.spent { + if n.spent || conflicts.contains(&n.txid) { continue; } let ready = self.note_ready(n, target, policy); @@ -1334,8 +2010,27 @@ impl WebWallet { _ => add(&mut orchard, &mut orchard_p, n.value_zat, ready), } } + let represented_transparent: std::collections::HashSet<_> = self + .utxos + .iter() + .map(|u| (u.txid.as_str(), u.index)) + .collect(); // Unmined sends/shields: change isn't in `notes` until the compact block is scanned. for (txid, agg) in &self.txs { + if conflicts.contains(txid) { + continue; + } + if !self.expired_unmined(agg) { + for (index, receipt) in &agg.accounted_transparent_receipts { + if receipt.builder + && receipt.observed_height.is_none_or(|height| height == 0) + && receipt.spent_in.is_none() + && !represented_transparent.contains(&(txid.as_str(), *index)) + { + t_p = t_p.saturating_add(receipt.value_zat); + } + } + } if agg.mined_height.is_some() || agg.received_zat == 0 || self.expired_unmined(agg) { continue; } @@ -1396,7 +2091,15 @@ impl WebWallet { u: &StoredUtxo, target: zcash_client_backend::data_api::wallet::TargetHeight, policy: &zcash_client_backend::data_api::wallet::ConfirmationsPolicy, + conflicts: &std::collections::HashSet, ) -> bool { + if conflicts.contains(&u.txid) { + return false; + } + #[cfg(feature = "transparent-pir")] + if self.pir_funding_unverified(&u.txid, u.index) { + return false; + } if self.transparent_scan_required && self .transparent_scan @@ -1405,7 +2108,7 @@ impl WebWallet { { return false; } - if u.coinbase { + if u.coinbase != CoinbaseStatus::NonCoinbase { return u.height > 0 && u.height.saturating_add(100) <= u32::from(target); } #[cfg(feature = "transparent-inputs")] @@ -1417,6 +2120,132 @@ impl WebWallet { tip > u.height && tip.saturating_sub(u.height) >= need } + #[cfg(feature = "transparent-pir")] + pub(crate) fn pir_funding_unverified(&self, txid: &str, index: u32) -> bool { + let key = format!("r:{txid}:{index}"); + self.transparent_pir + .unverified_outputs + .as_ref() + .map_or_else( + || { + self.transparent_pir.records.contains_key(&key) + || self.transparent_pir.projected_outputs.contains(&key) + }, + |outputs| outputs.contains(&key), + ) + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn pir_receipt_unaccounted(&self, txid: &str, index: u32) -> bool { + self.transparent_pir + .projected_outputs + .contains(&format!("r:{txid}:{index}")) + && !self + .txs + .get(txid) + .is_some_and(|agg| agg.accounted_transparent_receipts.contains_key(&index)) + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn record_public_transparent_output(&mut self, txid: &str, index: u32) { + // Publicly observed canonical rows no longer belong to PIR's deletion + // ownership. This alone does not discharge their funding block. + self.transparent_pir + .projected_outputs + .remove(&format!("r:{txid}:{index}")); + self.transparent_pir.projected_transactions.remove(txid); + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn confirm_public_transparent_output(&mut self, txid: &str, index: u32) { + self.record_public_transparent_output(txid, index); + if let Some(unverified) = &mut self.transparent_pir.unverified_outputs { + unverified.remove(&format!("r:{txid}:{index}")); + } + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn confirm_complete_public_transparent_outputs(&mut self) { + if self + .transparent_scan + .as_ref() + .is_none_or(|scan| scan.height != self.scanned_height) + { + return; + } + let state = &mut self.transparent_pir; + if let Some(unverified) = &mut state.unverified_outputs { + for output in &self.utxos { + let key = format!("r:{}:{}", output.txid, output.index); + // A public prefix already transferred deletion ownership. Full + // contiguous coverage to this tip now also validates its unspent state. + if output.height > 0 && !state.projected_outputs.contains(&key) { + unverified.remove(&key); + } + } + } + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn record_public_transparent_spend( + &mut self, + txid: &str, + previous: &str, + index: u32, + ) { + let Some(value) = self + .utxos + .iter() + .find(|u| { + u.txid == previous + && u.index == index + && u.spent + && u.spent_in.as_deref() == Some(txid) + }) + .map(|u| u.value_zat) + else { + return; + }; + if self.release_pir_spend_ownership(txid, previous, index) { + self.credit_transparent_debit(txid, previous, index, value); + } + } + + #[cfg(feature = "transparent-pir")] + pub(crate) fn release_pir_spend_ownership( + &mut self, + txid: &str, + previous: &str, + index: u32, + ) -> bool { + // The canonical identity excludes the publisher's claimed input position. + let prefix = format!("s:{txid}:"); + let suffix = format!(":{previous}:{index}"); + let claims: Vec<_> = self + .transparent_pir + .projected_spends + .range(prefix.clone()..) + .take_while(|key| key.starts_with(&prefix)) + .filter(|key| key.ends_with(&suffix)) + .cloned() + .collect(); + let owned = !claims.is_empty(); + for claim in claims { + self.transparent_pir.projected_spends.remove(&claim); + } + self.transparent_pir.projected_transactions.remove(txid); + owned + } + + fn transparent_pir_snapshot_status(&self) -> serde_json::Value { + #[cfg(feature = "transparent-pir")] + { + return self.transparent_pir_status(); + } + #[cfg(not(feature = "transparent-pir"))] + serde_json::Value::Null + } + pub fn wallet_snapshot(&self, server: &str) -> serde_json::Value { let b = self.balance(); let zip = zip321_uri(&self.unified_address, None).unwrap_or_default(); @@ -1433,6 +2262,7 @@ impl WebWallet { "transparentAddress": self.transparent_address, "zip321": zip, "scannedHeight": self.scanned_height, + "transparentPir": self.transparent_pir_snapshot_status(), "transparentScanHeight": self.transparent_scan.as_ref().map(|s| s.height), "transparentScanComplete": self.transparent_scan.as_ref().is_some_and(|s| s.height == self.scanned_height), "memoScanHeight": self.memo_scan.as_ref().map(|s| s.height), @@ -1961,6 +2791,22 @@ impl WebWallet { .collect(); let stays_pending = |txid: &Option| txid.as_ref().is_some_and(|t| unmined_sends.contains(t)); + for agg in self.txs.values_mut() { + for receipt in agg.accounted_transparent_receipts.values_mut() { + if receipt.observed_height.is_some_and(|height| height > keep) { + receipt.observed_height = None; + } + if let Some(spent) = &mut receipt.spent_in { + if spent.height.is_some_and(|height| height > keep) { + if unmined_sends.contains(&spent.txid) { + spent.height = None; + } else { + receipt.spent_in = None; + } + } + } + } + } self.notes.retain(|n| n.mined_height <= keep); for n in &mut self.notes { if n.spent_height.is_some_and(|h| h > keep) { @@ -2040,6 +2886,9 @@ impl WebWallet { self.restore_next_after_rewind(birthday_bound); } self.rewind_public_coverage(); + #[cfg(feature = "transparent-pir")] + self.transparent_pir.rewind(keep); + self.restore_pending_transparent_claims(); self.rebuild_trees()?; Ok(self.scanned_height) } @@ -2053,9 +2902,22 @@ impl WebWallet { if birthday > self.birthday { return Err(EngineError::Message("rescan later birthday".into())); } - if self.unmined_pending_txs().next().is_some() { + if self.pending_txs.keys().any(|id| { + self.txs + .get(&to_hex(id.as_ref())) + .is_some_and(|agg| agg.mined_height.is_none() && !self.expired_unmined(agg)) + }) { return Err(EngineError::Message("rescan pending payment".into())); } + if self + .output_locks + .iter() + .any(|lock| lock.expiry >= self.next_height()) + { + return Err(EngineError::Message( + "rescan active output reservation".into(), + )); + } let mut fresh = Self::from_ufvk(self.network, &self.ufvk, birthday, self.account_index)?; fresh.unified_address = self.unified_address.clone(); fresh.transparent_address = self.transparent_address.clone(); @@ -2064,6 +2926,11 @@ impl WebWallet { fresh.hardware = self.hardware.clone(); fresh.recent_recipients = self.recent_recipients.clone(); fresh.transparent_scan_required = self.transparent_scan_required; + #[cfg(feature = "transparent-pir")] + { + fresh.transparent_pir = self.transparent_pir.clone(); + fresh.transparent_pir.invalidate_projection(); + } *self = fresh; Ok(()) } @@ -2071,6 +2938,8 @@ impl WebWallet { /// Drop notes, trees, UTXOs, and history. Keep UFVK, birthday, and addresses. /// Next `sync` trial-decrypts from birthday. pub fn reset_scan(&mut self) { + #[cfg(feature = "transparent-pir")] + self.transparent_pir.invalidate_projection(); self.scan_error = None; self.invalidate_live_trees(); self.leaf_validation = Default::default(); @@ -2107,10 +2976,12 @@ impl WebWallet { /// can no longer be mined (zero never expires). /// Raw bytes of our sends still waiting to be mined, keyed like `txs`. fn unmined_pending_txs(&self) -> impl Iterator)> + '_ { - self.pending_txs.iter().filter_map(|(id, raw)| { + let conflicts = self.ineligible_pending_transactions(); + self.pending_txs.iter().filter_map(move |(id, raw)| { let key = to_hex(id.as_ref()); let agg = self.txs.get(&key)?; - (agg.mined_height.is_none() && !self.expired_unmined(agg)).then_some((*id, raw)) + (agg.mined_height.is_none() && !self.expired_unmined(agg) && !conflicts.contains(&key)) + .then_some((*id, raw)) }) } @@ -2129,9 +3000,15 @@ impl WebWallet { }) } - /// Hex of every unmined, unexpired send, for an idempotent rebroadcast. + /// Retained unmined bytes for deletion/rescan guards, including intents + /// that cannot currently be submitted. See `pending_submissions` for eligibility. pub fn pending_raw_txs(&self) -> Vec { - self.unmined_pending_txs() + self.retained_signed_txs() + .filter(|(id, _)| { + self.txs + .get(id) + .is_some_and(|agg| agg.mined_height.is_none()) + }) .map(|(_, raw)| to_hex(raw)) .collect() } @@ -2184,6 +3061,16 @@ impl WebWallet { .collect(); for txid in expired { let id = Some(txid.as_str()); + for agg in self.txs.values_mut() { + for receipt in agg.accounted_transparent_receipts.values_mut() { + if receipt.spent_in.as_ref().is_some_and(|spent| { + Some(spent.txid.as_str()) == id + && spent.height.is_none_or(|height| height == 0) + }) { + receipt.spent_in = None; + } + } + } for n in &mut self.notes { if n.spent_in.as_deref() == id && n.spent_height.is_none() { n.spent = false; @@ -2219,7 +3106,15 @@ impl WebWallet { let Some(agg) = self.txs.get(txid) else { return Ok(false); }; - if agg.mined_height.is_some() { + if agg.mined_height.is_some() + || self.txs.values().any(|agg| { + agg.accounted_transparent_receipts.values().any(|receipt| { + receipt.spent_in.as_ref().is_some_and(|spent| { + spent.txid == txid && spent.height.is_some_and(|height| height > 0) + }) + }) + }) + { return Err(EngineError::Message( "cannot abandon a mined transaction".into(), )); @@ -2243,6 +3138,13 @@ impl WebWallet { u.spent_in = None; } } + for agg in self.txs.values_mut() { + for receipt in agg.accounted_transparent_receipts.values_mut() { + if receipt.spent_in.as_ref().map(|spent| spent.txid.as_str()) == Some(txid) { + receipt.spent_in = None; + } + } + } if let Ok(id) = Self::txid_from_hex(txid) { self.pending_txs.remove(&id); } @@ -3447,6 +4349,11 @@ impl WebWallet { .filter(|u| u.spent) .map(|u| ((u.txid.clone(), u.index), u.spent_in.clone())) .collect(); + let prior_coinbase: std::collections::HashMap<_, _> = self + .utxos + .iter() + .map(|u| ((u.txid.clone(), u.index), u.coinbase)) + .collect(); let mut next = Vec::with_capacity(arr.len()); for item in arr { let txid = item @@ -3465,10 +4372,17 @@ impl WebWallet { let spent = spent_in.is_some() || pending_spent.contains_key(&(txid.to_string(), index)); next.push(StoredUtxo { - coinbase: item - .get("coinbase") - .and_then(|v| v.as_bool()) - .unwrap_or(false), + // Address-UTXO replies do not establish non-coinbase transaction + // structure. Preserve independently known classification and never + // clear a conservative coinbase restriction with a missing/false flag. + coinbase: if item.get("coinbase").and_then(|v| v.as_bool()) == Some(true) { + CoinbaseStatus::Coinbase + } else { + prior_coinbase + .get(&(txid.to_owned(), index)) + .copied() + .unwrap_or(CoinbaseStatus::Unknown) + }, txid: txid.to_string(), index, script: item @@ -3489,6 +4403,26 @@ impl WebWallet { spent_in, }); } + let incoming: Vec<_> = next + .iter() + .map(|u| (u.txid.clone(), u.index, u.value_zat)) + .collect(); + self.preflight_transparent_receipts(&incoming)?; + let credited: Vec<_> = next + .iter() + .filter(|u| { + #[cfg(feature = "transparent-pir")] + { + return u.height > 0 || !self.pir_funding_unverified(&u.txid, u.index); + } + #[cfg(not(feature = "transparent-pir"))] + { + let _ = u; + true + } + }) + .map(|u| (u.txid.clone(), u.index, u.value_zat, u.height)) + .collect(); let n = next.len() as u32; // The server lists only unspent outputs. Keep ours that a known wallet // transaction spent, so its history still shows the transparent input @@ -3499,16 +4433,33 @@ impl WebWallet { .utxos .iter() .filter(|u| { - u.spent + let known_spent = u.spent && u.spent_in .as_ref() - .is_some_and(|t| self.txs.contains_key(t)) - && !listed.contains(&(u.txid.clone(), u.index)) + .is_some_and(|t| self.txs.contains_key(t)); + #[cfg(feature = "transparent-pir")] + let retained_pir = self.pir_funding_unverified(&u.txid, u.index); + #[cfg(not(feature = "transparent-pir"))] + let retained_pir = false; + (known_spent || retained_pir) && !listed.contains(&(u.txid.clone(), u.index)) }) .cloned() .collect(); + #[cfg(feature = "transparent-pir")] + let confirmed: Vec<_> = next + .iter() + .filter(|u| u.height > 0) + .map(|u| (u.txid.clone(), u.index)) + .collect(); next.extend(kept); self.utxos = next; + for (txid, index, value, height) in credited { + self.credit_transparent_receipt(&txid, index, value, Some(height), false); + } + #[cfg(feature = "transparent-pir")] + for (txid, index) in confirmed { + self.confirm_public_transparent_output(&txid, index); + } // Explicit compatibility address lookup replaces compact coverage. A // future switch back must replay spends missed in this mode. self.transparent_scan_required = false; diff --git a/crates/z-engine/src/web/store_tests.rs b/crates/z-engine/src/web/store_tests.rs index df6cfca..0e3ee38 100644 --- a/crates/z-engine/src/web/store_tests.rs +++ b/crates/z-engine/src/web/store_tests.rs @@ -1361,6 +1361,7 @@ fn first_receipt_after_note_free_finalize_can_spend() { source.apply_utxos_json( r#"{"utxos":[{"txid":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ).unwrap(); + crate::web::scan_address_fixture_structure(&mut source); let unrelated = source.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); let parse = |raw: &[u8]| { Transaction::read( @@ -1523,6 +1524,7 @@ fn check_mined_self_send_history(blob: bool) { w.apply_utxos_json( r#"{"utxos":[{"txid":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ).unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shield = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); w.apply_mined_raw_tx(&shield, 1).unwrap(); w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); @@ -1593,6 +1595,7 @@ fn shield_history_keeps_its_transparent_input_through_refresh_and_mining() { w.apply_compact_blocks_blob(&super::super::encode_delimited([b1, b2])) .unwrap(); w.apply_utxos_json(UTXOS).unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shield = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); let txid = Transaction::read( shield.as_slice(), @@ -1631,6 +1634,7 @@ fn inputs_of_an_expired_unmined_send_become_spendable_again() { r#"{"utxos":[{"txid":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ) .unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shield = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); w.apply_mined_raw_tx(&shield, 1).unwrap(); let funded = w.balance().orchard_available; @@ -1741,6 +1745,7 @@ fn post_nu6_3_payments_use_ironwood_and_ironwood_notes_can_be_spent() { )) .unwrap(); // Funded before activation: an Orchard note and no Ironwood activity. + crate::web::scan_address_fixture_structure(&mut source); let shield = source.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); source.apply_mined_raw_tx(&shield, 1).unwrap(); recipient.apply_mined_raw_tx(&shield, 1).unwrap(); @@ -1838,6 +1843,7 @@ fn future_subtree_roots_cannot_move_scanned_anchor() { w.apply_utxos_json( r#"{"utxos":[{"txid":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ).unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shielded = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); w.apply_mined_raw_tx(&shielded, 1).unwrap(); w.finalize_scan_trees().unwrap(); @@ -2265,6 +2271,7 @@ fn funded_wallet() -> (WebWallet, u64) { r#"{"utxos":[{"txid":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ) .unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shield = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); w.apply_mined_raw_tx(&shield, 1).unwrap(); let funded = w.balance().orchard_available; @@ -2490,6 +2497,7 @@ fn zero_value_notes_do_not_block_spending() { w.apply_utxos_json( r#"{"utxos":[{"txid":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","index":0,"script":"76a914d3c0870e8e13a9ec320f1280889127aa15a4c0a188ac","valueZat":625000000,"height":1,"address":"tmV1zYhR2xisn6VWdCNKHpeD4S7L1U1nPH6"}]}"#, ).unwrap(); + crate::web::scan_address_fixture_structure(&mut w); let shield = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); w.apply_mined_raw_tx(&shield, 1).unwrap(); let own = w.unified_address().to_string(); diff --git a/crates/z-engine/src/web/transparent_pir.rs b/crates/z-engine/src/web/transparent_pir.rs new file mode 100644 index 0000000..291f9a4 --- /dev/null +++ b/crates/z-engine/src/web/transparent_pir.rs @@ -0,0 +1,4377 @@ +use super::*; +use crate::transparent_pir::{ + parse, refused, TransparentPirAnchor, TransparentPirCheckpoint, TransparentPirScript, +}; +use transparent_events::TransparentEvent; + +impl WebWallet { + pub(crate) fn reconcile_transparent_pir_history(&mut self) { + let owned: std::collections::BTreeSet<_> = self + .transparent_pir + .projected_spends + .iter() + .filter_map(|key| { + let fields: Vec<_> = key.split(':').collect(); + match fields.as_slice() { + ["s", spender, _, previous, index] => { + Some((*spender, *previous, index.parse::().ok()?)) + } + _ => None, + } + }) + .collect(); + let mut totals = std::collections::BTreeMap::<&str, (u64, u32)>::new(); + for output in &self.utxos { + let Some(spender) = output.spent_in.as_deref().filter(|_| output.spent) else { + continue; + }; + if owned.contains(&(spender, output.txid.as_str(), output.index)) { + let total = totals.entry(spender).or_default(); + total.0 = total.0.saturating_add(output.value_zat); + total.1 = total.1.saturating_add(1); + } + } + let mut receipts = std::collections::BTreeMap::<&str, (u64, u32)>::new(); + for output in &self.utxos { + if self.pir_receipt_unaccounted(&output.txid, output.index) { + let total = receipts.entry(output.txid.as_str()).or_default(); + total.0 = total.0.saturating_add(output.value_zat); + total.1 = total.1.saturating_add(1); + } + } + for (txid, agg) in &mut self.txs { + let (value, count) = totals.get(txid.as_str()).copied().unwrap_or_default(); + agg.pir_spent_zat = value; + agg.pir_spent_notes = count; + let (value, count) = receipts.get(txid.as_str()).copied().unwrap_or_default(); + if let Some(received) = &mut agg.pir_received_zat { + *received = value; + agg.pir_received_notes = count; + } + } + } + + /// The caller has accepted this exact transparent publisher profile. All scripts + /// and the target are obtained from this wallet; no host-provided amount is used. + pub fn begin_transparent_pir(&mut self, source_id: &str) -> Result { + if self.transparent_pir_rescan_required() { + return Err(refused("transparent_pir_rescan_required")); + } + let height = self.scanned_height; + let mut hash = super::super::from_hex( + self.hash_at(height) + .ok_or_else(|| refused("transparent_pir_anchor_unavailable"))?, + ) + .map_err(|_| refused("transparent_pir_anchor_unavailable"))?; + hash.reverse(); + let anchor = TransparentPirAnchor { + height: u64::from(height), + hash: super::super::to_hex(&hash), + }; + let scripts = self + .public_transparent_scripts()? + .into_iter() + .map(|(script, _)| TransparentPirScript { + script: super::super::to_hex(&script), + required_from: 0, + }) + .collect(); + let mut scope = self.transparent_pir.begin(source_id, anchor, scripts)?; + scope.accepted_headers.retain(|h| h.height != 0); + scope.accepted_headers.push(TransparentPirAnchor { + height: 0, + hash: self.network.genesis_hash().into(), + }); + self.transparent_pir.scope = Some(scope.clone()); + self.transparent_pir + .unverified_outputs + .get_or_insert_with(Default::default); + self.transparent_pir.wallet_cursor = Some(self.next_diversifier); + serde_json::to_string(&scope).map_err(|_| refused("transparent_pir_scope_invalid")) + } + + pub fn transparent_pir_checkpoint(&self) -> Option { + self.transparent_pir + .continuation + .as_ref() + .and_then(|c| serde_json::to_string(c).ok()) + } + + pub(crate) fn transparent_pir_rescan_required(&self) -> bool { + let legacy_receipts = self.transparent_pir.projected_outputs.iter().any(|key| { + key.split(':') + .nth(1) + .and_then(|id| self.txs.get(id)) + .is_some_and(|agg| agg.pir_received_zat.is_none()) + }); + self.legacy_transparent_receipt_accounting() + || legacy_receipts + || self.transparent_pir.unverified_outputs.is_none() + && (self.transparent_pir.bound_source.is_some() + || self.transparent_pir.scope.is_some() + || self.transparent_pir.checkpoint_id.is_some() + || !self.transparent_pir.records.is_empty() + || !self.transparent_pir.projected_outputs.is_empty()) + } + + pub fn transparent_pir_status(&self) -> serde_json::Value { + let mut state = self.transparent_pir.clone(); + let rescan_required = self.transparent_pir_rescan_required(); + if rescan_required { + state.complete = false; + } + if state + .scope + .as_ref() + .is_some_and(|scope| scope.anchor.height != u64::from(self.scanned_height)) + { + state.complete = false; + } + if state.wallet_cursor != Some(self.next_diversifier) { + state.complete = false; + } + if let Some(scope) = &state.scope { + let known = u32::try_from(scope.anchor.height) + .ok() + .and_then(|h| self.hash_at(h)) + .and_then(|h| super::super::from_hex(h).ok()) + .map(|mut h| { + h.reverse(); + super::super::to_hex(&h) + }); + if known.as_deref() != Some(&scope.anchor.hash) { + state.complete = false; + } + } + let mut status = state.status(); + status["rescanRequired"] = serde_json::json!(rescan_required); + let total: u64 = self + .utxos + .iter() + .filter(|u| { + !u.spent + && self + .transparent_pir + .records + .contains_key(&format!("r:{}:{}", u.txid, u.index)) + }) + .map(|u| u.value_zat) + .sum(); + status["confirmedZat"] = serde_json::json!(total.to_string()); + status + } + + pub fn apply_transparent_pir_checkpoint(&mut self, json: &str) -> Result { + self.apply_transparent_pir_checkpoint_inner(json, None) + } + + pub fn apply_transparent_pir_checkpoint_with_rewind( + &mut self, + json: &str, + rewind: z_transparent_pir_v2::AcceptedRewind<'_>, + ) -> Result { + self.apply_transparent_pir_checkpoint_inner(json, Some(rewind)) + } + + fn apply_transparent_pir_checkpoint_inner( + &mut self, + json: &str, + rewind: Option>, + ) -> Result { + if self.transparent_pir_rescan_required() { + return Err(refused("transparent_pir_rescan_required")); + } + let checkpoint: TransparentPirCheckpoint = parse(json)?; + let mut state = self.transparent_pir.clone(); + let witnessed = rewind.as_ref().is_some_and(|witness| { + state + .scope + .as_ref() + .is_some_and(|scope| witness.matches(scope, &checkpoint)) + }); + if rewind.is_some() && !witnessed { + return Err(refused("transparent_pir_rewind_evidence_invalid")); + } + let old_records = state.records.clone(); + let retirement_floor = state + .rewind_floor + .into_iter() + .chain(checkpoint.rewind_to.as_ref().map(|a| a.height as u32)) + .min(); + if let Some(floor) = retirement_floor { + state.prepare_rewind(floor); + } + if let Some(ancestor) = &checkpoint.rewind_to { + let height = u32::try_from(ancestor.height) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + let known = if height == 0 { + Some(self.network.genesis_hash().to_string()) + } else { + self.hash_at(height) + .map(|h| -> Result { + let mut h = super::super::from_hex(h) + .map_err(|_| refused("transparent_pir_rewind_anchor_unknown"))?; + h.reverse(); + Ok(super::super::to_hex(&h)) + }) + .transpose()? + }; + if known.as_deref() != Some(&ancestor.hash) && !(known.is_none() && witnessed) { + return Err(refused("transparent_pir_rewind_anchor_unknown")); + } + state.prepare_rewind(height); + } + let records = state.validate(&checkpoint)?; + let height = u32::try_from(checkpoint.anchor.height) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + let mut hash = super::super::from_hex(&checkpoint.anchor.hash) + .map_err(|_| refused("transparent_pir_anchor_invalid"))?; + hash.reverse(); + if self.hash_at(height) != Some(super::super::to_hex(&hash).as_str()) + || self.scanned_height != height + { + return Err(refused("transparent_pir_chain_changed")); + } + if self.transparent_pir.checkpoint_id.as_ref() == Some(&checkpoint.checkpoint_id) + && !self.transparent_pir.projection_invalidated + { + return serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")); + } + if retirement_floor.is_some() && !checkpoint.complete { + state.records = old_records; + state.stage(&checkpoint, &[])?; + state.rewind_floor = retirement_floor; + self.transparent_pir = state; + return serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")); + } + let scripts = self.public_transparent_scripts()?; + // Stage all canonical effects on clones. A malformed late record cannot + // partially mutate outputs, pending reservations or transaction history. + let mut utxos = self.utxos.clone(); + let mut txs = self.txs.clone(); + let mut next_diversifier = self.next_diversifier; + if let Some(floor) = retirement_floor { + let incoming: std::collections::BTreeMap<_, _> = records + .iter() + .map(|(_, e)| { + ( + crate::transparent_pir::event_key(*e), + crate::transparent_pir::digest(&e.to_bytes()), + ) + }) + .collect(); + let retired: Vec<_> = old_records + .iter() + .filter(|(key, (h, d))| *h > floor && incoming.get(*key) != Some(d)) + .map(|(k, _)| k.clone()) + .collect(); + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"s") && state.projected_spends.remove(key) { + let spending = Self::txid_from_hex(fields[1])?; + if self.pending_txs.contains_key(&spending) { + return Err(refused("transparent_pir_pending_reconciliation_required")); + } + let index: u32 = fields[4] + .parse() + .map_err(|_| refused("transparent_pir_state_invalid"))?; + if let Some(u) = utxos.iter_mut().find(|u| { + u.txid == fields[3] + && u.index == index + && u.spent_in.as_deref() == Some(fields[1]) + }) { + u.spent = false; + u.spent_in = None; + } + } + } + for key in &retired { + let fields: Vec<_> = key.split(':').collect(); + if fields.first() == Some(&"r") && state.projected_outputs.remove(key) { + let index: u32 = fields[2] + .parse() + .map_err(|_| refused("transparent_pir_state_invalid"))?; + if self + .output_locks + .iter() + .any(|l| l.txid == fields[1] && l.index == index && l.pool == "transparent") + || utxos.iter().any(|u| { + u.txid == fields[1] + && u.index == index + && u.spent_in.as_ref().is_some_and(|t| { + Self::txid_from_hex(t) + .ok() + .is_some_and(|id| self.pending_txs.contains_key(&id)) + }) + }) + { + return Err(refused("transparent_pir_pending_reconciliation_required")); + } + utxos.retain(|u| u.txid != fields[1] || u.index != index); + if let Some(unverified) = &mut state.unverified_outputs { + unverified.remove(key); + } + } + } + for id in &state.projected_transactions { + if !self.pending_txs.contains_key(&Self::txid_from_hex(id)?) { + if let Some(agg) = txs.get_mut(id) { + // Only PIR owns this inclusion stamp. Independent compact + // observations transfer that ownership before retirement; + // raw enhancement retains facts, but proves no inclusion. + if agg.mined_height.is_some_and(|h| h > floor) { + agg.mined_height = None; + } + } + } + } + } + for (script, event) in &records { + let (position, (_, address)) = scripts + .iter() + .enumerate() + .find(|(_, (s, _))| s == script) + .ok_or_else(|| refused("transparent_pir_scope_changed"))?; + match event { + TransparentEvent::Receive(e) => { + let txid = super::super::to_hex(&e.txid.0); + let mut mined = true; + if let Some(u) = utxos + .iter_mut() + .find(|u| u.txid == txid && u.index == e.output_index) + { + if u.value_zat != e.value + || u.script != super::super::to_hex(script) + || (u.height != 0 && u.height != e.height) + { + return Err(refused("transparent_pir_output_conflict")); + } + // Existing funding belongs to its original canonical source. + // PIR cannot mine an ordinary pending output or downgrade coinbase rules. + mined = u.height > 0; + } else { + state + .projected_outputs + .insert(crate::transparent_pir::event_key(*event)); + state + .unverified_outputs + .get_or_insert_with(Default::default) + .insert(crate::transparent_pir::event_key(*event)); + utxos.push(StoredUtxo { + txid: txid.clone(), + index: e.output_index, + script: super::super::to_hex(script), + value_zat: e.value, + height: e.height, + address: address.clone(), + coinbase: CoinbaseStatus::from_unverified(e.coinbase), + spent: false, + spent_in: None, + }); + } + if !txs.contains_key(&txid) { + state.projected_transactions.insert(txid.clone()); + } + let owned_receipt = state + .projected_outputs + .contains(&crate::transparent_pir::event_key(*event)); + let agg = txs.entry(txid).or_default(); + if owned_receipt { + agg.pir_received_zat.get_or_insert(0); + } + if agg.mined_height.is_some_and(|h| h != e.height) { + return Err(refused("transparent_pir_transaction_conflict")); + } + if mined { + agg.mined_height = Some(e.height); + } + next_diversifier = next_diversifier.max( + u32::try_from(position / 2 + 1) + .map_err(|_| refused("transparent_pir_scope_limit"))?, + ); + } + TransparentEvent::Spend(e) => { + let txid = super::super::to_hex(&e.spending_txid.0); + let previous = super::super::to_hex(&e.spent_txid.0); + if let Some(u) = utxos + .iter_mut() + .find(|u| u.txid == previous && u.index == e.spent_output_index) + { + if u.script != super::super::to_hex(script) + || (u.spent && u.spent_in.as_ref().is_some_and(|known| known != &txid)) + { + return Err(refused("transparent_pir_spend_conflict")); + } + if !u.spent { + state + .projected_spends + .insert(crate::transparent_pir::event_key(*event)); + } + u.spent = true; + u.spent_in = Some(txid.clone()); + } else if checkpoint.complete { + return Err(refused("transparent_pir_unresolved_spend")); + } + if !txs.contains_key(&txid) { + state.projected_transactions.insert(txid.clone()); + } + let agg = txs.entry(txid).or_default(); + if agg.mined_height.is_some_and(|h| h != e.height) { + return Err(refused("transparent_pir_transaction_conflict")); + } + agg.mined_height = Some(e.height); + } + } + } + let mut unspent = 0u64; + let mut spends = std::collections::BTreeMap::<&str, u64>::new(); + for u in &utxos { + let total = if let Some(spender) = u.spent_in.as_deref() { + spends.entry(spender).or_default() + } else if !u.spent { + &mut unspent + } else { + continue; + }; + *total = total + .checked_add(u.value_zat) + .ok_or_else(|| refused("transparent_pir_amount_invalid"))?; + zcash_protocol::value::Zatoshis::from_u64(*total) + .map_err(|_| refused("transparent_pir_amount_invalid"))?; + } + state.stage(&checkpoint, &records)?; + state.projection_invalidated = false; + // A received gap address expands inventory, so this checkpoint cannot + // assert completeness for the new scripts until a further pass covers them. + if next_diversifier != self.next_diversifier { + state.complete = false; + } + self.utxos = utxos; + self.txs = txs; + self.next_diversifier = next_diversifier; + self.transparent_pir = state; + self.recompute_pool_fields(); + if retirement_floor.is_some() { + self.txs.retain(|id, agg| { + !self.transparent_pir.projected_transactions.contains(id) + || agg.mined_height.is_some() + || agg.received_zat > 0 + || agg.spent_zat > 0 + || agg.received_notes > 0 + || agg.spent_notes > 0 + || agg.transparent_received > 0 + || agg.transparent_spent > 0 + || agg.has_shielded_activity() + || agg.history_metadata_complete + || agg.receipt_totals_recorded + || Self::txid_from_hex(id) + .ok() + .is_some_and(|id| self.pending_txs.contains_key(&id)) + }); + self.transparent_pir + .projected_transactions + .retain(|id| self.txs.contains_key(id)); + } + serde_json::to_string(&self.transparent_pir_status()) + .map_err(|_| refused("transparent_pir_status_invalid")) + } + + /// Prepare the exact persistence candidate without publishing its canonical + /// projection. The synchronous preparation owns only the fields this + /// checkpoint can modify; unrelated wallet state is never rolled back. + pub fn transparent_pir_checkpoint_snapshot(&mut self, json: &str) -> Result<(Vec, String)> { + self.transparent_pir_checkpoint_snapshot_inner(json, None) + } + + pub fn transparent_pir_checkpoint_snapshot_with_rewind( + &mut self, + json: &str, + rewind: z_transparent_pir_v2::AcceptedRewind<'_>, + ) -> Result<(Vec, String)> { + self.transparent_pir_checkpoint_snapshot_inner(json, Some(rewind)) + } + + fn transparent_pir_checkpoint_snapshot_inner( + &mut self, + json: &str, + rewind: Option>, + ) -> Result<(Vec, String)> { + let utxos = self.utxos.clone(); + let txs = self.txs.clone(); + let next_diversifier = self.next_diversifier; + let state = self.transparent_pir.clone(); + let result = (|| { + match rewind { + Some(witness) => { + self.apply_transparent_pir_checkpoint_with_rewind(json, witness)? + } + None => self.apply_transparent_pir_checkpoint(json)?, + }; + Ok((self.to_snapshot()?, self.wallet_snapshot("").to_string())) + })(); + self.utxos = utxos; + self.txs = txs; + self.next_diversifier = next_diversifier; + self.transparent_pir = state; + result + } + + pub fn acknowledge_transparent_pir_checkpoint(&mut self, id: &str) -> Result<()> { + self.transparent_pir.acknowledge(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::keys::{account_from_mnemonic, REGTEST_FAUCET_MNEMONIC}; + use crate::transparent_pir::{TransparentPirRecord, TransparentPirScope}; + use transparent_events::{ReceiveEvent, SpendEvent, Txid}; + use z_transparent_pir_v2::Coverage; + + pub(super) fn wallet() -> WebWallet { + let mut w = WebWallet::from_account( + account_from_mnemonic(REGTEST_FAUCET_MNEMONIC, Network::Regtest, 0).unwrap(), + 1, + ) + .unwrap(); + w.scanned_height = 3; + w.block_hashes.insert(3, "03".repeat(32)); + w + } + pub(super) fn checkpoint( + w: &mut WebWallet, + events: Vec, + ) -> TransparentPirCheckpoint { + let scope: TransparentPirScope = + serde_json::from_str(&w.begin_transparent_pir(&"ab".repeat(32)).unwrap()).unwrap(); + let script = super::super::super::to_hex(&w.public_transparent_scripts().unwrap()[0].0); + let through = u64::from(w.scanned_height); + TransparentPirCheckpoint { + source_id: scope.source_id, + generation: scope.generation, + scope_id: scope.scope_id, + anchor: scope.anchor, + checkpoint_id: "cd".repeat(32), + previous_checkpoint_id: None, + covered_through: through, + complete: true, + pending_pages: 0, + unresolved_spends: 0, + records: events + .into_iter() + .map(|e| TransparentPirRecord { + script: script.clone(), + event: super::super::super::to_hex(&e.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }) + .collect(), + coverage: scope + .scripts + .into_iter() + .map(|s| Coverage { + script: s.script, + required_from: s.required_from, + through, + revision: "ef".repeat(32), + }) + .collect(), + rewind_to: None, + continuation: String::new(), + } + } + pub(super) fn receive() -> TransparentEvent { + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 1, + txid: Txid([1; 32]), + transaction_index: 1, + output_index: 0, + value: 150000, + coinbase: false, + }) + } + pub(super) fn spend() -> TransparentEvent { + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 2, + spending_txid: Txid([2; 32]), + transaction_index: 1, + input_index: 0, + spent_txid: Txid([1; 32]), + spent_output_index: 0, + }) + } + fn public_range( + w: &mut WebWallet, + with_spend: bool, + ) -> Vec { + use zcash_client_backend::proto::compact_formats::{ + CompactBlock, CompactTx, CompactTxIn, TxOut, + }; + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + let mut blocks: Vec<_> = (1..=3) + .map(|h| CompactBlock { + height: h, + hash: vec![h as u8; 32], + prev_hash: vec![h as u8 - 1; 32], + vtx: vec![CompactTx { + index: 0, + txid: vec![h as u8 + 100; 32], + ..Default::default() + }], + ..Default::default() + }) + .collect(); + blocks[0].vtx.push(CompactTx { + index: 1, + txid: vec![1; 32], + vout: vec![TxOut { + value: 150000, + script_pub_key: script, + }], + ..Default::default() + }); + if with_spend { + blocks[1].vtx.push(CompactTx { + index: 1, + txid: vec![2; 32], + vin: vec![CompactTxIn { + prevout_txid: vec![1; 32], + prevout_index: 0, + }], + ..Default::default() + }); + } + blocks + } + fn apply_pir(w: &mut WebWallet, c: &TransparentPirCheckpoint) { + w.apply_transparent_pir_checkpoint(&serde_json::to_string(c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + } + fn replace_pir( + w: &mut WebWallet, + previous: String, + events: Vec, + ancestor: u64, + ) { + let mut c = checkpoint(w, events); + c.checkpoint_id = "de".repeat(32); + c.previous_checkpoint_id = Some(previous); + c.rewind_to = Some(TransparentPirAnchor { + height: ancestor, + hash: if ancestor == 0 { + Network::Regtest.genesis_hash().into() + } else { + format!("{ancestor:02x}").repeat(32) + }, + }); + apply_pir(w, &c); + } + fn private_candidate_fixture() -> (WebWallet, TransparentPirCheckpoint) { + let mut w = wallet(); + let mut last = receive(); + if let TransparentEvent::Receive(event) = &mut last { + event.txid = Txid([3; 32]); + event.height = 3; + event.value = 17000; + } + let cp = checkpoint(&mut w, vec![receive(), spend(), last]); + (w, cp) + } + + #[test] + fn checkpoint_candidate_keeps_live_projection_private_until_commit() { + let (mut w, cp) = private_candidate_fixture(); + let before = w.to_snapshot().unwrap(); + let input = serde_json::to_string(&cp).unwrap(); + let (bytes, preview) = w.transparent_pir_checkpoint_snapshot(&input).unwrap(); + assert!(w.to_snapshot().unwrap() == before); + assert!(w.history(20).is_empty()); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "0"); + let cold = WebWallet::from_snapshot(&bytes).unwrap(); + assert_eq!(cold.history(20).len(), 3); + assert_eq!(cold.transparent_pir_status()["confirmedZat"], "17000"); + // Importing snapshot bytes is the existing durable-owner boundary, + // so a persisted-but-unacknowledged candidate remains recoverable. + assert_eq!( + cold.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + assert_eq!( + serde_json::from_str::(&preview).unwrap()["transactions"] + .as_array() + .unwrap() + .len(), + 3 + ); + apply_pir(&mut w, &cp); + assert_eq!(w.history(20).len(), 3); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "17000"); + } + + #[test] + fn checkpoint_candidate_failure_preserves_exact_live_wallet() { + let (mut w, mut cp) = private_candidate_fixture(); + let before = w.to_snapshot().unwrap(); + cp.records[2].script = "ff".into(); + assert!(w + .transparent_pir_checkpoint_snapshot(&serde_json::to_string(&cp).unwrap()) + .is_err()); + assert!(w.to_snapshot().unwrap() == before); + } + + #[test] + fn checkpoint_commit_never_rolls_back_independent_writes_after_preparation() { + let (mut w, cp) = private_candidate_fixture(); + w.transparent_pir_checkpoint_snapshot(&serde_json::to_string(&cp).unwrap()) + .unwrap(); + let (script, address) = w.public_transparent_scripts().unwrap()[0].clone(); + w.apply_utxos_json(&serde_json::json!({"utxos":[{"txid":"08".repeat(32),"index":0,"valueZat":5000,"height":1,"script":super::super::super::to_hex(&script),"address":address}]}).to_string()).unwrap(); + apply_pir(&mut w, &cp); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.history(20).len(), 4); + assert_eq!(cold.transparent_pir_status()["confirmedZat"], "17000"); + assert_eq!( + cold.balance().transparent_available + cold.balance().transparent_pending, + 22000 + ); + assert!(cold + .utxos + .iter() + .any(|u| u.txid == "08".repeat(32) && u.value_zat == 5000 && !u.spent)); + } + + #[test] + fn header_progress_checkpoint_preserves_projection_reservation_and_pending_marker() { + use zcash_client_backend::data_api::OutputLockStore; + use zcash_client_backend::wallet::{LockOwner, OutputRef}; + use zcash_primitives::transaction::TxId as WalletTxId; + use zcash_protocol::PoolType; + + let mut w = wallet(); + let mut last_receive = receive(); + if let TransparentEvent::Receive(e) = &mut last_receive { + e.txid = Txid([3; 32]); + e.height = 3; + e.value = 17000; + } + let c = checkpoint(&mut w, vec![receive(), spend(), last_receive]); + apply_pir(&mut w, &c); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "17000"); + assert_eq!(w.history(20).len(), 3); + let owner = LockOwner::new([9; 32]); + w.lock_outputs( + &[OutputRef::new( + WalletTxId::from_bytes([3; 32]), + PoolType::TRANSPARENT, + 0, + )], + owner, + 100.into(), + ) + .unwrap(); + // A receipt-persistence marker, not a signed transaction or send fixture. + let pending_id = WalletTxId::from_bytes([9; 32]); + w.txs.entry("09".repeat(32)).or_default(); + w.pending_txs.insert(pending_id, vec![0xaa]); + let canonical = serde_json::to_value((&w.utxos, &w.txs)).unwrap(); + assert_eq!(w.output_locks.len(), 1); + let pending = w.pending_txs.clone(); + let mut progress = c.clone(); + progress.previous_checkpoint_id = Some(c.checkpoint_id); + progress.checkpoint_id = "de".repeat(32); + progress.complete = false; + progress.continuation = "opaque-verified-header-progress".into(); + apply_pir(&mut w, &progress); + assert_eq!(serde_json::to_value((&w.utxos, &w.txs)).unwrap(), canonical); + assert_eq!(w.output_locks.len(), 1); + assert_eq!(w.output_locks[0].expiry, 100); + assert_eq!( + w.active_lock_owner(&"03".repeat(32), "transparent", 0), + Some("09".repeat(32).as_str()) + ); + assert_eq!(w.pending_txs, pending); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "17000"); + assert_eq!(w.transparent_pir_status()["completion"], "incomplete"); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + serde_json::to_value((&restored.utxos, &restored.txs)).unwrap(), + canonical + ); + assert_eq!(restored.pending_txs, pending); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "17000"); + assert_eq!( + restored.transparent_pir_status()["completion"], + "incomplete" + ); + let saved: TransparentPirCheckpoint = + serde_json::from_str(&restored.transparent_pir_checkpoint().unwrap()).unwrap(); + assert_eq!(saved.continuation, progress.continuation); + } + #[test] + fn independently_verified_inputs_can_shield_after_pir_enrollment() { + for compact in [true, false] { + let mut w = wallet(); + let mut unverified = receive(); + if let TransparentEvent::Receive(e) = &mut unverified { + e.txid = Txid([3; 32]); + e.height = 3; + e.value = 70000; + } + let c = checkpoint(&mut w, vec![receive(), unverified]); + apply_pir(&mut w, &c); + assert!(w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).is_err()); + assert!(w.pending_txs.is_empty()); + if compact { + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + } else { + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + let rows: Vec<_> = w.utxos.iter().map(|u| serde_json::json!({ + "txid":u.txid,"index":u.index,"script":u.script, + "valueZat":u.value_zat,"height":if u.txid == "01".repeat(32) {1} else {0}, + "address":u.address + })).collect(); + w.apply_utxos_json(&serde_json::json!({"utxos":rows}).to_string()) + .unwrap(); + } + assert!(w.transparent_pir.scope.is_some()); + assert_eq!(w.balance().transparent_available, 150000); + assert_eq!(w.balance().transparent_pending, 70000); + let raw = w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).unwrap(); + assert!(raw.len() > 100); + assert!( + w.utxos + .iter() + .find(|u| u.txid == "01".repeat(32)) + .unwrap() + .spent + ); + assert!( + !w.utxos + .iter() + .find(|u| u.txid == "03".repeat(32)) + .unwrap() + .spent + ); + assert!(!w.pending_txs.is_empty()); + assert_eq!(w.balance().transparent_pending, 70000); + } + } + #[test] + fn independent_compact_receive_remains_ready_after_pir_enrollment() { + let mut w = wallet(); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!(w.balance().transparent_available, 150000); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + assert_eq!(w.balance().transparent_available, 150000); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 150000); + } + #[test] + fn pir_only_receive_waits_for_full_public_coverage_and_survives_pir_retirement() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + w.apply_transparent_blocks(&super::super::super::encode_delimited( + blocks[1..].iter().cloned(), + )) + .unwrap(); + assert_eq!(w.balance().transparent_available, 150000); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert_eq!(w.balance().transparent_available, 150000); + assert_eq!(w.history(20).len(), 1); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 150000); + } + #[test] + fn public_compact_spend_cannot_be_undone_by_pir_retirement() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, true); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + assert_eq!(w.balance().transparent_available, 0); + assert!(w.utxos[0].spent); + assert_eq!(w.history(20).len(), 2); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + assert!(restored.utxos[0].spent); + } + #[test] + fn legacy_false_or_absent_coinbase_is_unknown_through_refresh_and_wallet_write() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + for legacy in [None, Some(false), Some(true)] { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let mut saved: serde_json::Value = + serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + if let Some(value) = legacy { + saved["utxos"][0]["coinbase"] = serde_json::json!(value); + } else { + saved["utxos"][0] + .as_object_mut() + .unwrap() + .remove("coinbase"); + } + let mut restored = + WebWallet::from_snapshot(&serde_json::to_vec(&saved).unwrap()).unwrap(); + let u = restored.utxos[0].clone(); + let expected = CoinbaseStatus::from_unverified(legacy.unwrap_or(false)); + assert_eq!(u.coinbase, expected); + let row = serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script,"valueZat":u.value_zat,"height":u.height,"address":u.address,"coinbase":false}); + restored + .apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(restored.utxos[0].coinbase, expected); + assert_eq!(restored.balance().transparent_available, 0); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(150000).unwrap(), + transparent::address::Script(zcash_script::script::Code( + super::super::super::from_hex(&u.script).unwrap(), + )), + ), + Some(1.into()), + None, + None, + None, + ) + .unwrap(); + restored.put_received_transparent_utxo(&output).unwrap(); + assert_eq!(restored.utxos[0].coinbase, expected); + assert_eq!(restored.balance().transparent_available, 0); + let cold = WebWallet::from_snapshot(&restored.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.utxos[0].coinbase, expected); + assert_eq!(cold.balance().transparent_available, 0); + } + } + + #[test] + fn public_non_coinbase_classification_survives_refresh_and_cold_reopen() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!(w.utxos[0].coinbase, CoinbaseStatus::NonCoinbase); + let u = w.utxos[0].clone(); + let row = serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script,"valueZat":u.value_zat,"height":u.height,"address":u.address}); + w.apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(w.utxos[0].coinbase, CoinbaseStatus::NonCoinbase); + assert_eq!(w.balance().transparent_available, 150000); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.utxos[0].coinbase, CoinbaseStatus::NonCoinbase); + assert_eq!(cold.balance().transparent_available, 150000); + } + + #[test] + fn actual_full_transaction_classifies_unknown_zero_conf_without_pir() { + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + // Disposable structurally valid legacy non-coinbase bytes, not a consensus fixture. + let mut raw = 1u32.to_le_bytes().to_vec(); + raw.push(1); + raw.extend([7; 32]); + raw.extend(0u32.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + raw.push(1); + raw.extend(150000u64.to_le_bytes()); + raw.push(u8::try_from(script.len()).unwrap()); + raw.extend(script); + raw.extend(0u32.to_le_bytes()); + let tx = zcash_primitives::transaction::Transaction::read( + &raw[..], + zcash_protocol::consensus::BranchId::Nu6_3, + ) + .unwrap(); + let txid = super::super::super::to_hex(tx.txid().as_ref()); + let row = serde_json::json!({"txid":txid,"index":0,"script":super::super::super::to_hex(&w.public_transparent_scripts().unwrap()[0].0),"valueZat":150000,"height":0,"address":w.public_transparent_scripts().unwrap()[0].1}); + w.apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(w.utxos[0].coinbase, CoinbaseStatus::Unknown); + assert_eq!(w.balance().transparent_available, 0); + w.apply_raw_tx(&raw).unwrap(); + assert_eq!(w.utxos[0].coinbase, CoinbaseStatus::NonCoinbase); + assert_eq!(w.balance().transparent_available, 150000); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_available, 150000); + } + + #[test] + fn address_refresh_preserves_immature_pir_coinbase_maturity() { + let mut w = wallet(); + let mut reward = receive(); + if let TransparentEvent::Receive(event) = &mut reward { + event.coinbase = true; + } + let c = checkpoint(&mut w, vec![reward]); + apply_pir(&mut w, &c); + assert_eq!(w.balance().transparent_available, 0); + let u = w.utxos[0].clone(); + // Exact fields returned by GetAddressUtxos: coinbase is absent. + let listed = serde_json::json!({"utxos":[{ + "txid":u.txid,"index":u.index,"script":u.script, + "valueZat":u.value_zat,"height":u.height,"address":u.address + }]}); + w.apply_utxos_json(&listed.to_string()).unwrap(); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + assert!(w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).is_err()); + assert!(w.pending_txs.is_empty()); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + assert_eq!(restored.balance().transparent_pending, 150000); + w.scanned_height = 99; + assert_eq!(w.balance().transparent_available, 0); + w.scanned_height = 100; + assert_eq!(w.balance().transparent_available, 150000); + } + + #[test] + fn independent_address_refresh_promotes_pir_receive_without_releasing_pending_spend() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + // A public prefix establishes transaction structure, not complete unspent coverage. + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + + let u = w.utxos[0].clone(); + let listed = serde_json::json!({"utxos":[{"txid":u.txid,"index":u.index,"script":u.script,"valueZat":u.value_zat,"height":u.height,"address":u.address}]}); + w.apply_utxos_json(&listed.to_string()).unwrap(); + assert_eq!(w.balance().transparent_available, 150000); + w.utxos[0].spent = true; + w.utxos[0].spent_in = Some("09".repeat(32)); + w.txs.entry("09".repeat(32)).or_default(); + w.pending_txs.insert( + zcash_primitives::transaction::TxId::from_bytes([9; 32]), + vec![0xaa], + ); + let pending = w.pending_txs.clone(); + w.apply_utxos_json(&listed.to_string()).unwrap(); + assert_eq!(w.balance().transparent_available, 0); + assert!(w.utxos[0].spent); + assert_eq!( + w.utxos[0].spent_in.as_deref(), + Some("0909090909090909090909090909090909090909090909090909090909090909") + ); + assert!(w.pending_txs == pending); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert!(w.utxos[0].spent); + assert!(w.pending_txs == pending); + } + #[test] + fn confirmed_wallet_write_waits_for_transaction_structure_evidence() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let script = transparent::address::Script(zcash_script::script::Code( + super::super::super::from_hex(&w.utxos[0].script).unwrap(), + )); + for mined in [None, Some(1.into())] { + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(150000).unwrap(), + script.clone(), + ), + mined, + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!( + w.pir_funding_unverified(&"01".repeat(32), 0), + mined.is_none() + ); + } + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!(w.balance().transparent_available, 150000); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert_eq!(w.balance().transparent_available, 150000); + assert_eq!(w.history(20).len(), 1); + } + + #[test] + fn pir_does_not_promote_preexisting_unmined_or_coinbase_funding() { + for (height, coinbase) in [(0, false), (1, true)] { + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + w.apply_utxos_json( + &serde_json::json!({"utxos":[{ + "txid":"01".repeat(32),"index":0,"script":super::super::super::to_hex(&script), + "valueZat":150000,"height":height,"coinbase":coinbase + }]}) + .to_string(), + ) + .unwrap(); + let available = w.balance().transparent_available; + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + assert_eq!(w.utxos[0].height, height); + assert_eq!( + w.utxos[0].coinbase, + CoinbaseStatus::from_unverified(coinbase) + ); + assert_eq!(w.balance().transparent_available, available); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert_eq!(w.utxos[0].height, height); + assert_eq!(w.balance().transparent_available, available); + } + } + + #[test] + fn legacy_pir_snapshot_stays_unspendable_and_requires_explicit_rescan() { + let mut w = wallet(); + assert_eq!(w.transparent_pir_status()["rescanRequired"], false); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + assert_eq!(w.transparent_pir_status()["rescanRequired"], false); + let mut legacy: serde_json::Value = + serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + legacy["transparentPir"] + .as_object_mut() + .unwrap() + .remove("unverifiedOutputs"); + legacy["transparentPir"]["publicFundingIsolated"] = serde_json::json!(true); + let mut restored = WebWallet::from_snapshot(&serde_json::to_vec(&legacy).unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + assert_eq!(restored.transparent_pir_status()["rescanRequired"], true); + let before = restored.to_snapshot().unwrap(); + assert!(restored + .begin_transparent_pir(&"ab".repeat(32)) + .unwrap_err() + .to_string() + .contains("transparent_pir_rescan_required")); + assert_eq!(restored.to_snapshot().unwrap(), before); + assert!(restored + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap_err() + .to_string() + .contains("transparent_pir_rescan_required")); + assert_eq!(restored.to_snapshot().unwrap(), before); + let continuation = restored.transparent_pir_checkpoint(); + restored.rescan_from(1).unwrap(); + assert_eq!(restored.transparent_pir_status()["rescanRequired"], false); + assert!(restored.transparent_pir_checkpoint() == continuation); + restored.scanned_height = 3; + restored.block_hashes.insert(3, "03".repeat(32)); + apply_pir(&mut restored, &c); + assert_eq!(restored.balance().transparent_available, 0); + assert_eq!(restored.balance().transparent_pending, 150000); + } + + #[test] + fn explicit_rescan_preserves_pir_acquisition_and_issued_scripts() { + for birthday in [1, 2] { + let mut w = wallet(); + w.birthday = 2; + w.next_unified_address().unwrap(); + w.next_unified_address().unwrap(); + let scripts = w.public_transparent_scripts().unwrap(); + let address = w.unified_address.clone(); + let cursor = w.next_diversifier; + let mut c = checkpoint(&mut w, vec![receive()]); + c.continuation = "opaque-logical-cover-debt-and-acquisition".into(); + apply_pir(&mut w, &c); + let continuation = w.transparent_pir_checkpoint(); + let source = w.transparent_pir.bound_source.clone(); + w.rescan_from(birthday).unwrap(); + assert!(w.transparent_pir_checkpoint() == continuation); + assert_eq!(w.transparent_pir.bound_source, source); + assert_eq!(w.next_diversifier, cursor); + assert_eq!(w.unified_address, address); + assert_eq!(w.public_transparent_scripts().unwrap(), scripts); + assert_eq!(w.transparent_pir_status()["rescanRequired"], false); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + cold.scanned_height = 3; + cold.block_hashes.insert(3, "03".repeat(32)); + apply_pir(&mut cold, &c); + assert_eq!(cold.balance().transparent_available, 0); + assert_eq!(cold.balance().transparent_pending, 150000); + assert!(cold.transparent_pir_checkpoint() == continuation); + } + } + + #[test] + fn explicit_rescan_refuses_unknown_payment_and_active_reservation_atomically() { + let mut w = wallet(); + let mut c = checkpoint(&mut w, vec![receive()]); + c.continuation = "opaque-acquisition-and-cover-debt".into(); + apply_pir(&mut w, &c); + let continuation = w.transparent_pir_checkpoint(); + let pending = zcash_primitives::transaction::TxId::from_bytes([9; 32]); + w.pending_txs.insert(pending, vec![0xaa]); + w.txs.entry("09".repeat(32)).or_default().expiry_height = Some(100); + let before = w.to_snapshot().unwrap(); + assert!(w + .rescan_from(1) + .unwrap_err() + .to_string() + .contains("rescan pending payment")); + assert_eq!(w.to_snapshot().unwrap(), before); + assert_eq!(w.pending_txs.get(&pending), Some(&vec![0xaa])); + w.pending_txs.clear(); + w.txs.remove(&"09".repeat(32)); + w.output_locks.push(StoredLock { + txid: "01".repeat(32), + pool: "transparent".into(), + index: 0, + owner: "ff".repeat(32), + expiry: 100, + }); + let before = w.to_snapshot().unwrap(); + assert!(w + .rescan_from(1) + .unwrap_err() + .to_string() + .contains("rescan active output reservation")); + assert_eq!(w.to_snapshot().unwrap(), before); + assert_eq!(w.output_locks.len(), 1); + assert_eq!(w.output_locks[0].owner, "ff".repeat(32)); + assert!(w.transparent_pir_checkpoint() == continuation); + } + + #[test] + fn mined_wallet_write_on_spent_pir_row_cannot_authorize_retired_funding() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + apply_pir(&mut w, &c); + let script = transparent::address::Script(zcash_script::script::Code( + super::super::super::from_hex(&w.utxos[0].script).unwrap(), + )); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(40000).unwrap(), + script, + ), + Some(3.into()), + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + assert_eq!(w.utxos[0].value_zat, 150000); + assert_eq!(w.utxos[0].height, 1); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + assert_eq!(w.balance().transparent_available, 0); + let mut restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + let blocks = public_range(&mut restored, false); + restored + .apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!(restored.balance().transparent_available, 150000); + } + + #[test] + fn partial_public_receive_then_unmined_refresh_cannot_release_pir_funding() { + for height in [Some(0), None] { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + let u = w.utxos[0].clone(); + let mut row = serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script,"valueZat":u.value_zat,"address":u.address}); + if let Some(h) = height { + row["height"] = serde_json::json!(h); + } + w.apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert_eq!(w.utxos.len(), 1); + assert_eq!(w.balance().transparent_available, 0); + let mut restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + let blocks = public_range(&mut restored, false); + restored + .apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!(restored.balance().transparent_available, 150000); + } + } + + #[test] + fn partial_funding_block_survives_conflict_retirement_reset_and_cold_replay() { + let mut w = wallet(); + let mut c = checkpoint(&mut w, vec![receive()]); + c.continuation = "opaque-cover-debt-and-acquisition".into(); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + let mut conflict_receive = receive(); + if let TransparentEvent::Receive(ref mut receive) = conflict_receive { + receive.value = 200000; + } + let mut conflict = checkpoint(&mut w, vec![conflict_receive]); + conflict.checkpoint_id = "dc".repeat(32); + conflict.previous_checkpoint_id = Some(c.checkpoint_id.clone()); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&conflict).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert_eq!(w.balance().transparent_available, 0); + let continuation = w.transparent_pir_checkpoint(); + w.rescan_from(1).unwrap(); + assert!(w.transparent_pir_checkpoint() == continuation); + w.scanned_height = 3; + w.block_hashes.insert(3, "03".repeat(32)); + apply_pir(&mut w, &c); + assert_eq!(w.balance().transparent_available, 0); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert_eq!(w.utxos.len(), 1); + assert_eq!(w.balance().transparent_available, 0); + let mut restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + let u = restored.utxos[0].clone(); + let row = serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script,"valueZat":40000,"height":1}); + let before = restored.to_snapshot().unwrap(); + assert!(restored + .apply_utxos_json(&serde_json::json!({"utxos":[row.clone(),{}]}).to_string()) + .is_err()); + assert_eq!(restored.to_snapshot().unwrap(), before); + assert!(restored + .apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .is_err()); + assert!(restored.to_snapshot().unwrap() == before); + let row = serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script,"valueZat":150000,"height":1}); + restored + .apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(restored.balance().transparent_available, 150000); + assert_eq!(restored.utxos[0].value_zat, 150000); + } + + #[test] + fn unknown_zero_conf_funding_without_pir_waits_for_classification() { + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + w.apply_utxos_json( + &serde_json::json!({"utxos":[{ + "txid":"01".repeat(32),"index":0,"script":super::super::super::to_hex(&script), + "valueZat":150000,"height":0 + }]}) + .to_string(), + ) + .unwrap(); + assert_eq!(w.balance().transparent_available, 0); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_available, 0); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], false); + let c = checkpoint(&mut cold, vec![]); + apply_pir(&mut cold, &c); + assert_eq!(cold.balance().transparent_available, 0); + } + + #[test] + fn unmined_wallet_write_cannot_replace_pending_public_funding_fields() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, false); + w.apply_transparent_blocks(&super::super::super::encode_delimited(vec![ + blocks[0].clone() + ])) + .unwrap(); + let script = transparent::address::Script(zcash_script::script::Code( + super::super::super::from_hex(&w.utxos[0].script).unwrap(), + )); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(900000).unwrap(), + script, + ), + None, + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + assert_eq!(w.balance().transparent_available, 0); + w.apply_transparent_blocks(&super::super::super::encode_delimited( + blocks[1..].iter().cloned(), + )) + .unwrap(); + assert_eq!(w.balance().transparent_available, 150000); + assert_eq!(w.utxos[0].value_zat, 150000); + } + + #[test] + fn public_spend_authority_survives_retirement_of_conflicting_pir_input_index() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let mut claimed_spend = spend(); + if let TransparentEvent::Spend(ref mut spend) = claimed_spend { + spend.input_index = 1; + } + let c = checkpoint(&mut w, vec![receive(), claimed_spend]); + apply_pir(&mut w, &c); + let blocks = public_range(&mut w, true); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + assert_eq!(w.balance().transparent_available, 0); + assert!(w.utxos[0].spent); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.balance().transparent_available, 0); + assert!(restored.utxos[0].spent); + } + + #[test] + fn canonical_nonzero_receive_requires_durable_ack_and_survives_reopen() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!( + w.transparent_pir_status()["completion"], + "checkpoint-pending" + ); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + assert!(w + .acknowledge_transparent_pir_checkpoint(&"00".repeat(32)) + .is_err()); + let saved = w.to_snapshot().unwrap(); + let restored = WebWallet::from_snapshot(&saved).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!( + restored.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn reset_scan_replays_unchanged_checkpoint_without_discarding_acquisition_state() { + for spent in [false, true] { + let mut w = wallet(); + let events = if spent { + vec![receive(), spend()] + } else { + vec![receive()] + }; + let mut c = checkpoint(&mut w, events); + c.continuation = "opaque-acquisition-with-unpaid-cover-queries".into(); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + let saved = w.transparent_pir_checkpoint().unwrap(); + let expected = if spent { "0" } else { "150000" }; + assert_eq!(w.transparent_pir_status()["confirmedZat"], expected); + w.reset_scan(); + assert!(w.utxos.is_empty()); + assert!(w.history(20).is_empty()); + assert!(w.transparent_pir_checkpoint().unwrap() == saved); + // The shielded scanner has returned to the exact same accepted tip. + w.scanned_height = 3; + w.block_hashes.insert(3, "03".repeat(32)); + let mut reopened = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + reopened + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + reopened + .acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + assert_eq!(reopened.transparent_pir_status()["confirmedZat"], expected); + assert_eq!(reopened.history(20).len(), if spent { 2 } else { 1 }); + assert!(reopened.transparent_pir_checkpoint().unwrap() == saved); + } + } + + #[test] + fn pir_spend_generic_history_repairs_cold_legacy_and_same_checkpoint_replay() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive(), spend()]); + apply_pir(&mut w, &c); + let assert_spend = |w: &WebWallet| { + let rows = w.history(20); + let spent = rows.iter().find(|r| r.transparent_spent == 150000).unwrap(); + assert_eq!(spent.spent_zat, 150000); + assert_eq!(spent.sent_note_count, 1); + assert_eq!(spent.account_delta_zat, -150000); + }; + assert_spend(&w); + apply_pir(&mut w, &c); + assert_spend(&w); + let mut legacy: serde_json::Value = + serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + for agg in legacy["txs"].as_object_mut().unwrap().values_mut() { + agg.as_object_mut().unwrap().remove("pirSpentZat"); + agg.as_object_mut().unwrap().remove("pirSpentNotes"); + } + legacy["txs"]["02".repeat(32)]["spentZat"] = serde_json::json!(0); + legacy["txs"]["02".repeat(32)]["spentNotes"] = serde_json::json!(0); + let mut restored = WebWallet::from_snapshot(&serde_json::to_vec(&legacy).unwrap()).unwrap(); + assert_spend(&restored); + apply_pir(&mut restored, &c); + assert_spend(&restored); + let mut progress = c.clone(); + progress.previous_checkpoint_id = Some(c.checkpoint_id); + progress.checkpoint_id = "ce".repeat(32); + apply_pir(&mut restored, &progress); + assert_spend(&restored); + let mut reopened = WebWallet::from_snapshot(&restored.to_snapshot().unwrap()).unwrap(); + assert_spend(&reopened); + reopened.block_hashes.insert(1, "01".repeat(32)); + replace_pir(&mut reopened, progress.checkpoint_id, vec![receive()], 1); + assert!(!reopened.txs.contains_key(&"02".repeat(32))); + assert_eq!(reopened.history(20).len(), 1); + } + + #[test] + fn pir_spend_generic_history_preserves_independent_totals_on_retirement() { + let mut w = wallet(); + // Existing ordinary/full-transaction accounting, independently of the PIR input. + w.txs.insert( + "02".repeat(32), + crate::web::store::TxAgg { + mined_height: Some(2), + spent_zat: 700, + spent_notes: 2, + ..Default::default() + }, + ); + let c = checkpoint(&mut w, vec![receive(), spend()]); + apply_pir(&mut w, &c); + assert_eq!(w.txs[&"02".repeat(32)].spent_zat, 700); + assert_eq!( + w.history(20) + .iter() + .find(|r| r.transparent_spent == 150000) + .unwrap() + .spent_zat, + 150700 + ); + assert_eq!(w.txs[&"02".repeat(32)].spent_notes, 2); + assert_eq!( + w.history(20) + .iter() + .find(|r| r.transparent_spent == 150000) + .unwrap() + .sent_note_count, + 3 + ); + apply_pir(&mut w, &c); + let mut legacy: serde_json::Value = + serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + let agg = legacy["txs"]["02".repeat(32)].as_object_mut().unwrap(); + agg.remove("pirSpentZat"); + agg.remove("pirSpentNotes"); + agg.insert("spentZat".into(), serde_json::json!(700)); + agg.insert("spentNotes".into(), serde_json::json!(2)); + let mut restored = WebWallet::from_snapshot(&serde_json::to_vec(&legacy).unwrap()).unwrap(); + assert_eq!(restored.txs[&"02".repeat(32)].spent_zat, 700); + assert_eq!( + restored + .history(20) + .iter() + .find(|r| r.transparent_spent == 150000) + .unwrap() + .spent_zat, + 150700 + ); + assert_eq!(restored.txs[&"02".repeat(32)].spent_notes, 2); + assert_eq!( + restored + .history(20) + .iter() + .find(|r| r.transparent_spent == 150000) + .unwrap() + .sent_note_count, + 3 + ); + restored.block_hashes.insert(1, "01".repeat(32)); + replace_pir(&mut restored, c.checkpoint_id, vec![receive()], 1); + assert_eq!(restored.txs[&"02".repeat(32)].spent_zat, 700); + assert_eq!(restored.txs[&"02".repeat(32)].spent_notes, 2); + assert_eq!(restored.txs[&"02".repeat(32)].transparent_spent, 0); + let reopened = WebWallet::from_snapshot(&restored.to_snapshot().unwrap()).unwrap(); + assert_eq!(reopened.txs[&"02".repeat(32)].spent_zat, 700); + assert_eq!(reopened.txs[&"02".repeat(32)].spent_notes, 2); + } + + #[test] + fn pir_spend_generic_history_transfers_to_public_write_without_doubling() { + use zcash_client_backend::data_api::{SentTransaction, WalletWrite}; + use zcash_client_backend::proto::compact_formats::{CompactTx, CompactTxIn, TxOut}; + use zcash_primitives::transaction::Transaction; + use zcash_protocol::{ + consensus::{BlockHeight, BranchId}, + value::Zatoshis, + }; + for rewound in [false, true] { + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + let address = w.public_transparent_scripts().unwrap()[0].1.clone(); + w.apply_utxos_json( + &serde_json::json!({"utxos":[{ + "txid":"03".repeat(32),"index":0,"script":super::super::super::to_hex(&script), + "valueZat":700,"height":1,"address":address + }]}) + .to_string(), + ) + .unwrap(); + // Structurally valid disposable bytes, with inert script signatures. + // This exercises WalletWrite accounting, not consensus or signing. + let mut raw = 1u32.to_le_bytes().to_vec(); + raw.push(2); + for previous in [1, 3] { + raw.extend([previous; 32]); + raw.extend(0u32.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + } + raw.push(1); + raw.extend(140700u64.to_le_bytes()); + raw.push(u8::try_from(script.len()).unwrap()); + raw.extend(&script); + raw.extend(0u32.to_le_bytes()); + let tx = Transaction::read(&raw[..], BranchId::for_height(&Network::Regtest, 2.into())) + .unwrap(); + let spender = super::super::super::to_hex(tx.txid().as_ref()); + let mut event = spend(); + if let TransparentEvent::Spend(e) = &mut event { + e.spending_txid = Txid(*tx.txid().as_ref()); + } + let c = checkpoint(&mut w, vec![receive(), event]); + apply_pir(&mut w, &c); + assert_eq!( + (w.txs[&spender].spent_zat, w.txs[&spender].spent_notes), + (0, 0) + ); + assert_eq!( + w.history(20) + .iter() + .find(|r| r.transparent_spent == 150000) + .unwrap() + .spent_zat, + 150000 + ); + if rewound { + w.block_hashes.insert(1, "01".repeat(32)); + w.rewind_to_height_checked(1).unwrap(); + assert!(w.utxos.iter().all(|u| !u.spent)); + // The synthetic shielded scanner catches back up to the same accepted tip. + w.scanned_height = 3; + w.block_hashes.insert(3, "03".repeat(32)); + } + let sent = SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + BlockHeight::from_u32(2).into(), + 0, + &[], + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + w.store_transactions_to_be_sent(&[sent]).unwrap(); + assert_eq!( + (w.txs[&spender].spent_zat, w.txs[&spender].spent_notes), + (150700, 2) + ); + let pending = w.pending_txs.clone(); + let mut blocked = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + blocked.block_hashes.insert(1, "01".repeat(32)); + let mut retirement = checkpoint(&mut blocked, vec![]); + retirement.checkpoint_id = "de".repeat(32); + retirement.previous_checkpoint_id = Some(c.checkpoint_id.clone()); + retirement.rewind_to = Some(TransparentPirAnchor { + height: 0, + hash: Network::Regtest.genesis_hash().into(), + }); + let before = blocked.to_snapshot().unwrap(); + assert!(blocked + .apply_transparent_pir_checkpoint(&serde_json::to_string(&retirement).unwrap()) + .unwrap_err() + .to_string() + .contains("transparent_pir_pending_reconciliation_required")); + assert_eq!(blocked.to_snapshot().unwrap(), before); + assert_eq!(blocked.pending_txs, pending); + let mut blocks = public_range(&mut w, false); + blocks[0].vtx.push(CompactTx { + index: 2, + txid: vec![3; 32], + vout: vec![TxOut { + value: 700, + script_pub_key: script.clone(), + }], + ..Default::default() + }); + blocks[1].vtx.push(CompactTx { + index: 1, + txid: tx.txid().as_ref().to_vec(), + vin: [1, 3] + .into_iter() + .map(|b| CompactTxIn { + prevout_txid: vec![b; 32], + prevout_index: 0, + }) + .collect(), + vout: vec![TxOut { + value: 140700, + script_pub_key: script, + }], + ..Default::default() + }); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + assert_eq!( + (w.txs[&spender].spent_zat, w.txs[&spender].spent_notes), + (150700, 2) + ); + assert_eq!( + ( + w.txs[&spender].pir_spent_zat, + w.txs[&spender].pir_spent_notes + ), + (0, 0) + ); + w.block_hashes.insert(1, "01".repeat(32)); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + assert_eq!( + (w.txs[&spender].spent_zat, w.txs[&spender].spent_notes), + (150700, 2) + ); + assert_eq!(w.pending_txs, pending); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + (cold.txs[&spender].spent_zat, cold.txs[&spender].spent_notes), + (150700, 2) + ); + assert_eq!(cold.pending_txs, pending); + } + } + + #[cfg(feature = "native")] + #[test] + fn retirement_keeps_independent_enhancement_of_pir_created_transaction() { + use zcash_primitives::transaction::Transaction; + use zcash_protocol::consensus::BranchId; + let mut w = wallet(); + w.scanned_height = 1_000_001; + w.block_hashes.insert(1_000_001, "03".repeat(32)); + let recipient = WebWallet::from_account( + account_from_mnemonic( + "legal winner thank year wave sausage worth useful legal winner thank yellow", + Network::Regtest, + 0, + ) + .unwrap(), + 1, + ) + .unwrap(); + let raw = crate::web::shielded_memo_transaction( + &w, + &recipient, + "independent outgoing", + 50000, + true, + ); + let tx = + Transaction::read(&raw[..], BranchId::for_height(&Network::Regtest, 4.into())).unwrap(); + let id = super::super::super::to_hex(tx.txid().as_ref()); + let mut event = spend(); + if let TransparentEvent::Spend(e) = &mut event { + e.spending_txid = Txid(*tx.txid().as_ref()); + } + let c = checkpoint(&mut w, vec![receive(), event]); + apply_pir(&mut w, &c); + assert!(w.transparent_pir.projected_transactions.contains(&id)); + w.enhance_raw_tx(&raw).unwrap(); + assert_eq!(w.txs[&id].outgoing_shielded_zat, 50000); + assert_eq!(w.txs[&id].memos, ["independent outgoing"]); + w.block_hashes.insert(1, "01".repeat(32)); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + let rows = w.history(20); + let row = rows + .iter() + .find(|row| row.txid == WebWallet::display_txid(&id)) + .expect("independently recovered outgoing history survives PIR retirement"); + assert_eq!(row.outgoing_shielded_zat, 50000); + assert_eq!(row.memos, ["independent outgoing"]); + assert_eq!( + row.mined_height, None, + "raw enhancement does not prove inclusion" + ); + assert_eq!((row.spent_zat, row.sent_note_count), (0, 0)); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.txs[&id].outgoing_shielded_zat, 50000); + } + + #[test] + fn retirement_removes_pure_pir_receive_history_and_memo_work() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + assert_eq!(w.history(20)[0].received_zat, 150000); + assert_eq!(w.history(20)[0].received_note_count, 1); + replace_pir(&mut w, c.checkpoint_id, vec![], 0); + assert!( + w.history(20).is_empty(), + "retired publisher-only receive has no canonical history" + ); + assert!(w.memo_enhancement_txids(20).is_empty()); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold.history(20).is_empty()); + assert!(cold.memo_enhancement_txids(20).is_empty()); + } + + #[test] + fn retirement_keeps_independent_compact_spend_before_pool_rebuild() { + use prost::Message; + use zcash_client_backend::proto::compact_formats::{ + ChainMetadata, CompactBlock, CompactSaplingSpend, CompactTx, + }; + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive(), spend()]); + apply_pir(&mut w, &c); + let spender = "02".repeat(32); + assert!(w.transparent_pir.projected_transactions.contains(&spender)); + // A retained owned note is independently consumed by the actual compact scanner. + // This is scan/accounting coverage, not a transaction consensus fixture. + w.notes.push(crate::web::store::TrackedNote { + pool: "sapling".into(), + txid: "09".repeat(32), + output_index: 0, + value_zat: 700, + nf: "11".repeat(32), + is_change: false, + spent: false, + spent_in: None, + spent_height: None, + mined_height: 1, + }); + let b = CompactBlock { + height: 4, + hash: vec![4; 32], + prev_hash: vec![3; 32], + chain_metadata: Some(ChainMetadata { + sapling_commitment_tree_size: 0, + orchard_commitment_tree_size: 0, + ironwood_commitment_tree_size: 0, + }), + vtx: vec![CompactTx { + txid: vec![2; 32], + spends: vec![CompactSaplingSpend { nf: vec![0x11; 32] }], + ..Default::default() + }], + ..Default::default() + }; + w.apply_compact_block(&b.encode_to_vec()).unwrap(); + assert_eq!( + (w.txs[&spender].spent_zat, w.txs[&spender].spent_notes), + (700, 1) + ); + assert_eq!( + w.txs[&spender].sapling_spent, 0, + "pool rebuild has not happened yet" + ); + w.block_hashes.insert(1, "01".repeat(32)); + replace_pir(&mut w, c.checkpoint_id, vec![receive()], 1); + let row = w + .history(20) + .into_iter() + .find(|r| r.txid == WebWallet::display_txid(&spender)) + .unwrap(); + assert_eq!( + (row.spent_zat, row.sent_note_count, row.sapling_spent), + (700, 1, 700) + ); + assert_eq!(row.mined_height, Some(4)); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.txs[&spender].spent_zat, 700); + } + + #[test] + fn old_pir_receipt_accounting_requires_explicit_rescan_without_losing_acquisition() { + let mut w = wallet(); + let mut c = checkpoint(&mut w, vec![receive(), spend()]); + c.continuation = "opaque-durable-acquisition-with-cover-debt".into(); + apply_pir(&mut w, &c); + let saved = w.transparent_pir_checkpoint(); + let scripts = w.public_transparent_scripts().unwrap(); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + let agg = old["txs"]["01".repeat(32)].as_object_mut().unwrap(); + agg.remove("pirReceivedZat"); + agg.remove("pirReceivedNotes"); + agg.insert("receivedZat".into(), serde_json::json!(150000)); + agg.insert("receivedNotes".into(), serde_json::json!(1)); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], true); + assert_eq!(cold.transparent_pir_checkpoint(), saved); + let before = cold.to_snapshot().unwrap(); + assert!(cold + .begin_transparent_pir(&"ab".repeat(32)) + .unwrap_err() + .to_string() + .contains("transparent_pir_rescan_required")); + assert!(cold + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap_err() + .to_string() + .contains("transparent_pir_rescan_required")); + assert_eq!(cold.to_snapshot().unwrap(), before); + cold.rescan_from(1).unwrap(); + assert_eq!(cold.transparent_pir_checkpoint(), saved); + assert_eq!(cold.public_transparent_scripts().unwrap(), scripts); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], false); + cold.scanned_height = 3; + cold.block_hashes.insert(3, "03".repeat(32)); + apply_pir(&mut cold, &c); + apply_pir(&mut cold, &c); + let rows = cold.history(20); + assert_eq!(rows.len(), 2); + let receive = rows + .iter() + .find(|r| r.transparent_received == 150000) + .unwrap(); + assert_eq!( + (receive.received_zat, receive.received_note_count), + (150000, 1) + ); + assert_eq!(cold.transparent_pir_checkpoint(), saved); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], false); + } + + #[test] + fn constructed_transparent_receipt_transfers_only_its_accounting_once() { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::Recipient; + use zcash_primitives::transaction::Transaction; + use zcash_protocol::{ + consensus::{BlockHeight, BranchId}, + value::Zatoshis, + }; + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + let address = w.public_transparent_scripts().unwrap()[0].1.clone(); + w.apply_utxos_json(&serde_json::json!({"utxos":[{"txid":"09".repeat(32),"index":0,"script":super::super::super::to_hex(&script),"valueZat":27000,"height":1,"address":address}]}).to_string()).unwrap(); + // Parsed legacy-format disposable transaction; inert script signatures. + let mut raw = 1u32.to_le_bytes().to_vec(); + raw.push(1); + raw.extend([9; 32]); + raw.extend(0u32.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + raw.push(1); + raw.extend(17000u64.to_le_bytes()); + raw.push(script.len() as u8); + raw.extend(&script); + raw.extend(0u32.to_le_bytes()); + let tx = + Transaction::read(&raw[..], BranchId::for_height(&Network::Regtest, 3.into())).unwrap(); + let id = super::super::super::to_hex(tx.txid().as_ref()); + let mut event = receive(); + if let TransparentEvent::Receive(e) = &mut event { + e.txid = Txid(*tx.txid().as_ref()); + e.height = 3; + e.value = 17000; + } + let c = checkpoint(&mut w, vec![event]); + apply_pir(&mut w, &c); + let output = SentTransactionOutput::from_parts( + 0, + Recipient::InternalTransparent { + receiving_account: 0, + recipient_address: tx.transparent_bundle().unwrap().vout[0] + .recipient_address() + .unwrap(), + }, + Zatoshis::from_u64(17000).unwrap(), + None, + ); + let outputs = [output]; + let sent = SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + BlockHeight::from_u32(3).into(), + 0, + &outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + w.store_transactions_to_be_sent(&[sent]).unwrap(); + w.recompute_pool_fields(); + let row = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (17000, 1)); + assert!( + w.pir_funding_unverified(&id, 0), + "construction is not funding confirmation" + ); + let listed = w.utxos.iter().find(|u| u.txid == id).unwrap(); + let reply=serde_json::json!({"utxos":[{"txid":id,"index":0,"script":listed.script,"height":3,"valueZat":17000,"address":listed.address}]}).to_string(); + w.apply_utxos_json(&reply).unwrap(); + w.apply_utxos_json(&reply).unwrap(); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (17000, 1)); + } + + fn builder_accounting_fixture() -> (WebWallet, zcash_primitives::transaction::Transaction) { + let mut w = wallet(); + let (script, address) = w.public_transparent_scripts().unwrap()[0].clone(); + w.apply_utxos_json(&serde_json::json!({"utxos":[{"txid":"09".repeat(32),"index":0,"script":super::super::super::to_hex(&script),"valueZat":27000,"height":1,"address":address}]}).to_string()).unwrap(); + // Actual parsed transaction/callback metadata; these inert script signatures + // are not a consensus-valid transaction or an enabled Web TEX proposal. + let mut raw = 1u32.to_le_bytes().to_vec(); + raw.push(1); + raw.extend([9; 32]); + raw.extend(0u32.to_le_bytes()); + raw.push(0); + raw.extend(u32::MAX.to_le_bytes()); + raw.push(1); + raw.extend(17000u64.to_le_bytes()); + raw.push(script.len() as u8); + raw.extend(script); + raw.extend(0u32.to_le_bytes()); + let tx = zcash_primitives::transaction::Transaction::read( + &raw[..], + zcash_protocol::consensus::BranchId::for_height(&Network::Regtest, 3.into()), + ) + .unwrap(); + (w, tx) + } + + fn builder_accounting_store_result( + w: &mut WebWallet, + tx: &zcash_primitives::transaction::Transaction, + ephemeral: bool, + ) -> std::result::Result<(), String> { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::Recipient; + use zcash_protocol::{consensus::BlockHeight, value::Zatoshis}; + let address = tx.transparent_bundle().unwrap().vout[0] + .recipient_address() + .unwrap(); + let recipient = if ephemeral { + Recipient::EphemeralTransparent { + receiving_account: 0, + ephemeral_address: address, + outpoint: transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + } + } else { + Recipient::InternalTransparent { + receiving_account: 0, + recipient_address: address, + } + }; + let outputs = [SentTransactionOutput::from_parts( + 0, + recipient, + tx.transparent_bundle().unwrap().vout[0].value(), + None, + )]; + let sent = SentTransaction::new( + tx, + std::time::UNIX_EPOCH.into(), + BlockHeight::from_u32(3).into(), + 0, + &outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + w.store_transactions_to_be_sent(&[sent]) + } + + fn builder_accounting_store( + w: &mut WebWallet, + tx: &zcash_primitives::transaction::Transaction, + ephemeral: bool, + ) { + builder_accounting_store_result(w, tx, ephemeral).unwrap(); + } + + #[test] + fn builder_accounting_pending_transparent_change_survives_missing_utxo_and_cold_refresh() { + for ephemeral in [false, true] { + let (mut w, tx) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &tx, ephemeral); + assert!(w + .utxos + .iter() + .all(|u| u.txid != super::super::super::to_hex(tx.txid().as_ref()))); + let assert_pending = |w: &WebWallet| { + assert_eq!(w.balance().transparent_pending, 17000); + assert_eq!(w.balance().orchard_pending, 0); + assert_eq!(w.balance().total_pending, 17000); + let row = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received + ), + (17000, 1, 17000) + ); + assert_eq!(w.pending_raw_txs().len(), 1); + }; + assert_pending(&w); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_pending(&cold); + cold.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_pending(&cold); + let output = zcash_client_backend::wallet::WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + tx.transparent_bundle().unwrap().vout[0].clone(), + None, + None, + None, + None, + ) + .unwrap(); + zcash_client_backend::data_api::WalletWrite::put_received_transparent_utxo( + &mut cold, &output, + ) + .unwrap(); + assert_pending(&cold); + cold.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_pending(&cold); + } + } + + #[test] + fn builder_accounting_public_credit_then_callback_is_idempotent() { + for ephemeral in [false, true] { + let (mut w, tx) = builder_accounting_fixture(); + let mut event = receive(); + if let TransparentEvent::Receive(e) = &mut event { + e.txid = Txid(*tx.txid().as_ref()); + e.height = 3; + e.value = 17000; + } + let c = checkpoint(&mut w, vec![event]); + apply_pir(&mut w, &c); + let id = super::super::super::to_hex(tx.txid().as_ref()); + let u = w.utxos.iter().find(|u| u.txid == id).unwrap(); + let reply = serde_json::json!({"utxos":[{"txid":u.txid,"index":u.index,"script":u.script,"height":3,"valueZat":17000,"address":u.address}]}).to_string(); + w.apply_utxos_json(&reply).unwrap(); + builder_accounting_store(&mut w, &tx, ephemeral); + builder_accounting_store(&mut w, &tx, ephemeral); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received + ), + (17000, 1, 17000) + ); + assert!(!cold.pir_funding_unverified(&id, 0)); + } + } + + #[test] + fn builder_accounting_public_wallet_write_reconciles_history_immediately() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + apply_pir(&mut w, &c); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new([1; 32], 0), + transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(150000).unwrap(), + transparent::address::Script(zcash_script::script::Code( + super::super::super::from_hex(&w.utxos[0].script).unwrap(), + )), + ), + Some(1.into()), + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + // Observe the public history immediately, without a repair/recompute call. + let row = w.history(20).remove(0); + assert_eq!((row.received_zat, row.received_note_count), (150000, 1)); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + ( + cold.history(20)[0].received_zat, + cold.history(20)[0].received_note_count + ), + (150000, 1) + ); + } + + #[test] + fn builder_accounting_ordinary_public_credit_then_callback_is_idempotent() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + for ephemeral in [false, true] { + let (mut w, tx) = builder_accounting_fixture(); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + tx.transparent_bundle().unwrap().vout[0].clone(), + Some(3.into()), + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + builder_accounting_store(&mut w, &tx, ephemeral); + builder_accounting_store(&mut w, &tx, ephemeral); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (17000, 1)); + } + } + + #[test] + fn builder_accounting_represented_spent_change_is_not_resurrected_as_pending() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + use zcash_primitives::transaction::{Authorized, TransactionData}; + use zcash_protocol::{consensus::BranchId, value::Zatoshis}; + let (mut w, tx) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &tx, false); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + tx.transparent_bundle().unwrap().vout[0].clone(), + None, + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + let child = TransactionData::::from_parts_v6( + BranchId::Nu6_3, + 0, + 20.into(), + Some(transparent::bundle::Bundle { + vin: vec![transparent::bundle::TxIn::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + transparent::address::Script::default(), + u32::MAX, + )], + vout: vec![transparent::bundle::TxOut::new( + Zatoshis::from_u64(7000).unwrap(), + tx.transparent_bundle().unwrap().vout[0] + .script_pubkey() + .clone(), + )], + authorization: transparent::bundle::Authorized, + }), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &child, false); + assert_eq!(w.balance().transparent_pending, 7000); + assert!(w + .utxos + .iter() + .any(|u| u.txid == super::super::super::to_hex(tx.txid().as_ref()) && u.spent)); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_pending, 7000); + assert_eq!(cold.pending_raw_txs().len(), 2); + use zcash_client_backend::proto::compact_formats::{CompactTx, CompactTxIn, TxOut}; + let mut blocks = public_range(&mut cold, false); + blocks[0].vtx.push(CompactTx { + index: 2, + txid: tx.txid().as_ref().to_vec(), + vout: vec![TxOut { + value: 17000, + script_pub_key: tx.transparent_bundle().unwrap().vout[0] + .script_pubkey() + .0 + .0 + .clone(), + }], + ..Default::default() + }); + blocks[1].vtx.push(CompactTx { + index: 2, + txid: child.txid().as_ref().to_vec(), + vin: vec![CompactTxIn { + prevout_txid: tx.txid().as_ref().to_vec(), + prevout_index: 0, + }], + vout: vec![TxOut { + value: 7000, + script_pub_key: child.transparent_bundle().unwrap().vout[0] + .script_pubkey() + .0 + .0 + .clone(), + }], + ..Default::default() + }); + cold.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + builder_accounting_store(&mut cold, &child, false); + let parent_id = super::super::super::to_hex(tx.txid().as_ref()); + assert_eq!( + cold.txs() + .get(&parent_id) + .unwrap() + .accounted_transparent_receipts[&0] + .spent_in + .as_ref() + .unwrap() + .height, + Some(2) + ); + cold.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + let cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + // The independently observed child receipt was omitted, and the spent + // parent stays represented. Neither historical receipt becomes pending. + assert_eq!( + cold.balance().transparent_pending + cold.balance().transparent_available, + 0 + ); + } + + #[test] + fn builder_accounting_expired_transparent_change_is_not_pending() { + use zcash_primitives::transaction::{Authorized, TransactionData}; + use zcash_protocol::consensus::BranchId; + let (mut w, template) = builder_accounting_fixture(); + let tx = TransactionData::::from_parts_v6( + BranchId::Nu6_3, + 0, + 4.into(), + template.transparent_bundle().cloned(), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &tx, false); + assert_eq!(w.balance().transparent_pending, 17000); + let block = zcash_client_backend::proto::compact_formats::CompactBlock { + height: 4, + hash: vec![4; 32], + prev_hash: vec![3; 32], + ..Default::default() + }; + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([block])) + .unwrap(); + // Ordinary expiry releases the original27000 input; the17000 change + // metadata remains history and does not become an additional balance. + assert_eq!(w.balance().transparent_pending, 27000); + assert!(w.pending_raw_txs().is_empty()); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_pending, 27000); + } + + #[cfg(feature = "native")] + fn builder_mixed_fixture( + store: bool, + ) -> (WebWallet, zcash_primitives::transaction::Transaction) { + builder_mixed_fixture_pool(store, true) + } + + #[cfg(feature = "native")] + fn builder_mixed_fixture_pool( + store: bool, + ironwood: bool, + ) -> (WebWallet, zcash_primitives::transaction::Transaction) { + use zcash_primitives::transaction::{Authorized, Transaction, TransactionData}; + use zcash_protocol::consensus::BranchId; + let (mut w, transparent) = builder_accounting_fixture(); + let raw = crate::web::shielded_memo_transaction(&w, &w, "mixed receipt", 30000, ironwood); + let shielded = Transaction::read(&raw[..], BranchId::Nu6_3).unwrap(); + let tx = if ironwood { + TransactionData::::from_parts_v6( + BranchId::Nu6_3, + 0, + 20.into(), + transparent.transparent_bundle().cloned(), + None, + None, + shielded.ironwood_bundle().cloned(), + ) + } else { + TransactionData::::from_parts( + zcash_primitives::transaction::TxVersion::V5, + BranchId::for_height(&Network::Regtest, 4.into()), + 0, + 20.into(), + transparent.transparent_bundle().cloned(), + None, + None, + shielded.orchard_bundle().cloned(), + ) + } + .freeze() + .unwrap(); + if store { + store_mixed_builder_receipt(&mut w, &tx); + } + (w, tx) + } + + #[cfg(feature = "native")] + fn store_mixed_builder_receipt( + w: &mut WebWallet, + tx: &zcash_primitives::transaction::Transaction, + ) { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::{Note as WalletNote, Recipient}; + use zcash_protocol::{consensus::BlockHeight, value::Zatoshis}; + let keys = std::collections::HashMap::from([(0u32, w.decode_ufvk().unwrap())]); + let decrypted = zcash_client_backend::decrypt_transaction( + &Network::Regtest, + None, + Some(BlockHeight::from_u32(3)), + &tx, + &keys, + ); + let note = decrypted + .ironwood_outputs() + .first() + .or_else(|| decrypted.orchard_outputs().first()) + .unwrap(); + let outputs = [ + SentTransactionOutput::from_parts( + 0, + Recipient::InternalTransparent { + receiving_account: 0, + recipient_address: tx.transparent_bundle().unwrap().vout[0] + .recipient_address() + .unwrap(), + }, + Zatoshis::from_u64(17000).unwrap(), + None, + ), + SentTransactionOutput::from_parts( + note.index(), + Recipient::InternalShielded { + receiving_account: 0, + external_address: None, + note: Box::new(WalletNote::Orchard { + note: note.note().0.clone(), + pool: note.note().1, + }), + }, + Zatoshis::from_u64(30000).unwrap(), + None, + ), + ]; + let sent = SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + BlockHeight::from_u32(3).into(), + 0, + &outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + w.store_transactions_to_be_sent(&[sent]).unwrap(); + } + + #[cfg(feature = "native")] + #[test] + fn builder_accounting_mixed_pending_receipts_stay_in_their_pools() { + let (mut w, tx) = builder_mixed_fixture(true); + for wallet in [ + &w, + &WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(), + ] { + let balance = wallet.balance(); + assert_eq!( + ( + balance.transparent_pending, + balance.ironwood_pending, + balance.orchard_pending, + balance.total_pending + ), + (17000, 30000, 0, 47000) + ); + } + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + let mut block = w.mined_raw_tx_block(&raw, 1).unwrap(); + block.prev_hash = vec![3; 32]; + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([block])) + .unwrap(); + assert_eq!( + w.balance().transparent_pending, + 17000, + "shielded inclusion is not transparent outpoint materialization" + ); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_pending, 17000); + store_mixed_builder_receipt(&mut w, &tx); + let row = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + (row.received_zat, row.received_note_count, row.mined_height), + (47000, 2, Some(4)) + ); + } + + #[cfg(feature = "native")] + #[test] + fn durable_receipt_raw_mixed_credit_then_callback_is_single() { + let (mut w, tx) = builder_mixed_fixture(false); + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + w.apply_raw_tx(&raw).unwrap(); + w.apply_raw_tx(&raw).unwrap(); + let row = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received, + row.ironwood_received + ), + (47000, 2, 17000, 30000) + ); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_eq!( + w.balance().transparent_pending + w.balance().transparent_available, + 0 + ); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + store_mixed_builder_receipt(&mut cold, &tx); + store_mixed_builder_receipt(&mut cold, &tx); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received, + row.ironwood_received + ), + (47000, 2, 17000, 30000) + ); + } + + #[test] + fn durable_receipt_ordinary_credit_survives_omission_before_callback() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let (mut w, tx) = builder_accounting_fixture(); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + tx.transparent_bundle().unwrap().vout[0].clone(), + None, + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + builder_accounting_store(&mut cold, &tx, false); + builder_accounting_store(&mut cold, &tx, false); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!((row.received_zat, row.received_note_count), (17000, 1)); + } + + #[test] + fn durable_receipt_legacy_pir_ambiguity_refuses_callback_before_mutation() { + let (mut w, tx) = builder_accounting_fixture(); + let mut event = receive(); + if let TransparentEvent::Receive(e) = &mut event { + e.txid = Txid(*tx.txid().as_ref()); + e.height = 3; + e.value = 17000; + } + let c = checkpoint(&mut w, vec![event]); + apply_pir(&mut w, &c); + let id = super::super::super::to_hex(tx.txid().as_ref()); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][&id]["receivedZat"] = 17000.into(); + old["txs"][&id]["receivedNotes"] = 1.into(); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("pirReceivedZat"); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], true); + let before = cold.to_snapshot().unwrap(); + assert!( + builder_accounting_store_result(&mut cold, &tx, false).is_err(), + "ambiguous legacy credit must refuse atomically" + ); + assert!( + cold.to_snapshot().unwrap() == before, + "legacy refusal must preserve the entire wallet" + ); + } + + #[test] + fn durable_receipt_two_indices_and_late_batch_conflict_are_atomic() { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::{Recipient, WalletTransparentOutput}; + use zcash_primitives::transaction::{Authorized, TransactionData}; + use zcash_protocol::{consensus::BranchId, value::Zatoshis}; + let (mut w, template) = builder_accounting_fixture(); + let output = template.transparent_bundle().unwrap().vout[0].clone(); + let mut bundle = template.transparent_bundle().unwrap().clone(); + bundle.vout.push(transparent::bundle::TxOut::new( + Zatoshis::from_u64(3000).unwrap(), + output.script_pubkey().clone(), + )); + let tx = TransactionData::::from_parts_v6( + BranchId::Nu6_3, + 0, + 20.into(), + Some(bundle), + None, + None, + None, + ) + .freeze() + .unwrap(); + for index in 0..2 { + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), index), + tx.transparent_bundle().unwrap().vout[index as usize].clone(), + None, + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + } + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let recipient = output.recipient_address().unwrap(); + let make = |index, value| { + SentTransactionOutput::from_parts( + index, + Recipient::InternalTransparent { + receiving_account: 0, + recipient_address: recipient, + }, + Zatoshis::from_u64(value).unwrap(), + None, + ) + }; + let outputs = [make(0, 17000), make(1, 3000)]; + fn callback<'a>( + tx: &'a zcash_primitives::transaction::Transaction, + outputs: &'a [SentTransactionOutput], + ) -> SentTransaction<'a, u32> { + SentTransaction::new( + tx, + std::time::UNIX_EPOCH.into(), + 3.into(), + 0, + outputs, + Zatoshis::from_u64(7000).unwrap(), + &[], + ) + } + let good = callback(&tx, &outputs); + let bad_outputs = [make(1, 4000)]; + let bad = callback(&tx, &bad_outputs); + let before = cold.to_snapshot().unwrap(); + assert!(cold.store_transactions_to_be_sent(&[good, bad]).is_err()); + assert!( + cold.to_snapshot().unwrap() == before, + "batch must preflight before saving pending bytes or spending inputs" + ); + for _ in 0..2 { + cold.store_transactions_to_be_sent(&[callback(&tx, &outputs)]) + .unwrap(); + } + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received + ), + (20000, 2, 20000) + ); + let before = cold.to_snapshot().unwrap(); + let id = super::super::super::to_hex(tx.txid().as_ref()); + assert!(cold + .apply_utxos_json( + &serde_json::json!({"utxos":[{"txid":id,"index":1,"valueZat":4000,"height":1}]}) + .to_string() + ) + .is_err()); + assert!(cold.to_snapshot().unwrap() == before); + } + + #[cfg(feature = "native")] + #[test] + fn durable_receipt_builder_inclusion_rewind_and_expiry_keep_claim_pending_only() { + let (mut w, tx) = builder_mixed_fixture(true); + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + let mut block = w.mined_raw_tx_block(&raw, 0).unwrap(); + block.prev_hash = vec![3; 32]; + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([block])) + .unwrap(); + assert_eq!(w.balance().transparent_pending, 17000); + w.rewind_to_height(3).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().transparent_pending, 17000); + store_mixed_builder_receipt(&mut cold, &tx); + assert_eq!(cold.balance().transparent_pending, 17000); + assert_eq!( + cold.txs() + .get(&super::super::super::to_hex(tx.txid().as_ref())) + .unwrap() + .mined_height, + None + ); + assert!(cold.abandon_unmined(&tx.txid().to_string()).unwrap()); + assert_eq!(cold.balance().transparent_pending, 27000); + } + + #[cfg(feature = "native")] + #[test] + fn durable_receipt_clean_legacy_shielded_callback_still_works() { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::{Note, Recipient}; + use zcash_protocol::value::Zatoshis; + let mut w = wallet(); + let raw = crate::web::shielded_memo_transaction(&w, &w, "legacy control", 30000, true); + let tx = zcash_primitives::transaction::Transaction::read( + &raw[..], + zcash_protocol::consensus::BranchId::Nu6_3, + ) + .unwrap(); + let key = w.decode_ufvk().unwrap(); + let mut keys = std::collections::HashMap::new(); + keys.insert(0, key); + let decrypted = zcash_client_backend::decrypt_transaction( + &w.network(), + None, + Some(3.into()), + &tx, + &keys, + ); + let note = decrypted + .ironwood_outputs() + .iter() + .find(|o| { + matches!( + o.transfer_type(), + zcash_client_backend::TransferType::Incoming + | zcash_client_backend::TransferType::AccountInternal + ) + }) + .unwrap(); + let outputs = [SentTransactionOutput::from_parts( + note.index(), + Recipient::InternalShielded { + receiving_account: 0, + external_address: None, + note: Box::new(Note::Orchard { + note: note.note().0.clone(), + pool: orchard::ValuePool::Ironwood, + }), + }, + Zatoshis::from_u64(30000).unwrap(), + None, + )]; + let id = super::super::super::to_hex(tx.txid().as_ref()); + w.txs_mut().entry(id.clone()).or_default(); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("receiptAccountingLegacy"); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + let callback = || { + SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + 3.into(), + 0, + &outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ) + }; + cold.store_transactions_to_be_sent(&[callback()]).unwrap(); + cold.store_transactions_to_be_sent(&[callback()]).unwrap(); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.ironwood_received + ), + (30000, 1, 30000) + ); + } + + #[test] + fn durable_receipt_confirmed_materialization_does_not_revive_after_omission() { + use zcash_client_backend::{data_api::WalletWrite, wallet::WalletTransparentOutput}; + let (mut w, tx) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &tx, true); + let output = WalletTransparentOutput::from_parts( + transparent::bundle::OutPoint::new(*tx.txid().as_ref(), 0), + tx.transparent_bundle().unwrap().vout[0].clone(), + Some(1.into()), + None, + None, + None, + ) + .unwrap(); + w.put_received_transparent_utxo(&output).unwrap(); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + cold.balance().transparent_pending + cold.balance().transparent_available, + 0 + ); + builder_accounting_store(&mut cold, &tx, true); + assert_eq!( + cold.balance().transparent_pending + cold.balance().transparent_available, + 0 + ); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.received_zat, + row.received_note_count, + row.transparent_received + ), + (17000, 1, 17000) + ); + assert!(cold + .txs() + .get(&super::super::super::to_hex(tx.txid().as_ref())) + .unwrap() + .mined_height + .is_some()); + } + + #[test] + fn durable_receipt_mapless_legacy_batch_refusal_preserves_all_intent() { + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::Recipient; + use zcash_protocol::value::Zatoshis; + let (w, tx) = builder_accounting_fixture(); + let id = super::super::super::to_hex(tx.txid().as_ref()); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + // Genuine old schema shape: historical independent receipt totals, without + // a per-index receipt ledger or a current accounting marker. + old["txs"][&id] = serde_json::to_value(TxAgg { + received_zat: 17000, + received_notes: 1, + transparent_received: 17000, + ..Default::default() + }) + .unwrap(); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("accountedTransparentReceipts"); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("receiptAccountingLegacy"); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + let goodtx = zcash_primitives::transaction::TransactionData::< + zcash_primitives::transaction::Authorized, + >::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 21.into(), + tx.transparent_bundle().cloned(), + None, + None, + None, + ) + .freeze() + .unwrap(); + assert_ne!(goodtx.txid(), tx.txid()); + let output = |transaction: &zcash_primitives::transaction::Transaction| { + SentTransactionOutput::from_parts( + 0, + Recipient::InternalTransparent { + receiving_account: 0, + recipient_address: transaction.transparent_bundle().unwrap().vout[0] + .recipient_address() + .unwrap(), + }, + Zatoshis::from_u64(17000).unwrap(), + None, + ) + }; + let good_outputs = [output(&goodtx)]; + let bad_outputs = [output(&tx)]; + let good = SentTransaction::new( + &goodtx, + std::time::UNIX_EPOCH.into(), + 3.into(), + 0, + &good_outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + let bad = SentTransaction::new( + &tx, + std::time::UNIX_EPOCH.into(), + 3.into(), + 0, + &bad_outputs, + Zatoshis::from_u64(10000).unwrap(), + &[], + ); + let before = cold.to_snapshot().unwrap(); + assert!(cold.store_transactions_to_be_sent(&[good, bad]).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + assert!(cold.pending_raw_txs().is_empty()); + } + + #[cfg(feature = "native")] + #[test] + fn durable_receipt_omitted_input_callback_inclusion_rewind_retains_signed_intent() { + let (mut w, tx) = builder_mixed_fixture(false); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert!(w.utxos.is_empty()); + store_mixed_builder_receipt(&mut w, &tx); + assert_eq!(w.pending_raw_txs().len(), 1); + let row = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + (row.spent_zat, row.sent_note_count, row.transparent_spent), + (27000, 1, 27000) + ); + assert_eq!(w.balance().transparent_pending, 17000); + let mut raw = Vec::new(); + tx.write(&mut raw).unwrap(); + let mut block = w.mined_raw_tx_block(&raw, 0).unwrap(); + block.prev_hash = vec![3; 32]; + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([block])) + .unwrap(); + assert_eq!(w.pending_raw_txs().len(), 0); + assert!(w.pending_txs.contains_key(&tx.txid())); + w.rewind_to_height(3).unwrap(); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + cold.pending_raw_txs().len(), + 1, + "exact saved signed transaction must survive compact rewind despite omitted input row" + ); + assert_eq!(cold.balance().transparent_pending, 17000); + let row = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert_eq!( + (row.spent_zat, row.sent_note_count, row.transparent_spent), + (27000, 1, 27000) + ); + assert_eq!( + cold.history_matching(1, None, Some(&tx.txid().to_string())) + .len(), + 1 + ); + } + + fn confirmed_conflict_fixture() -> ( + WebWallet, + zcash_primitives::transaction::Transaction, + zcash_primitives::transaction::Transaction, + ) { + use zcash_primitives::transaction::{Authorized, TransactionData}; + let (mut w, original) = builder_accounting_fixture(); + w.scanned_height = 0; + w.block_hashes.clear(); + let mut chain = public_range(&mut w, false); + for block in &mut chain { + block.vtx.clear(); + } + w.apply_compact_blocks_blob(&super::super::super::encode_delimited(chain)) + .unwrap(); + builder_accounting_store(&mut w, &original, false); + let mut bundle = original.transparent_bundle().unwrap().clone(); + bundle.vout[0] = transparent::bundle::TxOut::new( + bundle.vout[0].value(), + transparent::address::TransparentAddress::PublicKeyHash([5; 20]) + .script() + .into(), + ); + let replacement = TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 10.into(), + Some(bundle), + None, + None, + None, + ) + .freeze() + .unwrap(); + (w, original, replacement) + } + + fn include_conflicting_spender( + w: &mut WebWallet, + replacement: &zcash_primitives::transaction::Transaction, + ) { + include_conflicting_spenders(w, &[(1, replacement)]); + } + + fn include_conflicting_spenders( + w: &mut WebWallet, + replacements: &[(usize, &zcash_primitives::transaction::Transaction)], + ) { + let blocks = public_spender_blocks(w, replacements); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + } + + fn public_spender_blocks( + w: &mut WebWallet, + replacements: &[(usize, &zcash_primitives::transaction::Transaction)], + ) -> Vec { + use zcash_client_backend::proto::compact_formats::{CompactTx, CompactTxIn, TxOut}; + let mut blocks = public_range(w, false); + blocks[0].vtx.pop(); + for (block_index, replacement) in replacements { + blocks[*block_index].vtx.push(CompactTx { + index: 1, + txid: replacement.txid().as_ref().to_vec(), + vin: replacement + .transparent_bundle() + .unwrap() + .vin + .iter() + .map(|input| CompactTxIn { + prevout_txid: input.prevout().hash().to_vec(), + prevout_index: input.prevout().n(), + }) + .collect(), + vout: replacement + .transparent_bundle() + .unwrap() + .vout + .iter() + .map(|output| TxOut { + value: u64::from(output.value()), + script_pub_key: output.script_pubkey().0 .0.clone(), + }) + .collect(), + ..Default::default() + }); + } + blocks + } + + #[test] + fn confirmed_conflict_preserves_signed_intent_but_stops_submission_and_pending_change() { + let (mut w, original, replacement) = confirmed_conflict_fixture(); + assert_eq!(w.pending_raw_txs().len(), 1); + assert_eq!(w.balance().transparent_pending, 17000); + include_conflicting_spender(&mut w, &replacement); + let before = w.to_snapshot().unwrap(); + assert!(builder_accounting_store_result(&mut w, &original, false).is_err()); + assert!(w.to_snapshot().unwrap() == before); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold.pending_txs.contains_key(&original.txid())); + assert!( + cold.pending_submissions().is_empty(), + "a different confirmed spender prevents rebroadcast of the saved intent" + ); + assert_eq!(cold.balance().total_pending, 0); + let row = cold + .history_matching(1, None, Some(&original.txid().to_string())) + .remove(0); + assert_eq!((row.spent_zat, row.sent_note_count), (27000, 1)); + assert_eq!(row.mined_height, None); + } + + #[test] + fn confirmed_conflict_does_not_contribute_unmined_change_balance() { + let (mut w, _original, replacement) = confirmed_conflict_fixture(); + include_conflicting_spender(&mut w, &replacement); + assert_eq!(w.balance().transparent_pending, 0); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().total_pending, 0); + } + + #[test] + fn confirmed_conflict_rewind_restores_saved_claims_and_rescan_cannot_drop_intent() { + let (mut w, original, replacement) = confirmed_conflict_fixture(); + let saved = w.pending_txs[&original.txid()].clone(); + include_conflicting_spender(&mut w, &replacement); + let before = w.to_snapshot().unwrap(); + assert!(w + .rescan_from(1) + .unwrap_err() + .to_string() + .contains("rescan pending payment")); + assert!(w.to_snapshot().unwrap() == before); + assert_eq!(w.rewind_to_height(1).unwrap(), 1); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.pending_txs[&original.txid()], saved); + assert_eq!(cold.pending_submissions().len(), 1); + assert_eq!(cold.balance().transparent_pending, 17000); + let original_id = super::super::super::to_hex(original.txid().as_ref()); + let input = cold + .utxos + .iter() + .find(|u| u.txid == "09".repeat(32)) + .unwrap(); + assert!(input.spent); + assert_eq!(input.spent_in.as_deref(), Some(original_id.as_str())); + assert!(cold.abandon_unmined(&original.txid().to_string()).unwrap()); + assert_eq!(cold.balance().transparent_pending, 27000); + assert!(!cold.pending_txs.contains_key(&original.txid())); + } + + #[cfg(feature = "native")] + #[test] + fn confirmed_conflict_suppresses_mixed_pool_pending_outputs() { + let (mut w, original) = builder_mixed_fixture(true); + let mut bundle = original.transparent_bundle().unwrap().clone(); + bundle.vout[0] = transparent::bundle::TxOut::new( + bundle.vout[0].value(), + transparent::address::TransparentAddress::PublicKeyHash([5; 20]) + .script() + .into(), + ); + let replacement = zcash_primitives::transaction::TransactionData::< + zcash_primitives::transaction::Authorized, + >::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 10.into(), + Some(bundle), + None, + None, + None, + ) + .freeze() + .unwrap(); + include_conflicting_spender(&mut w, &replacement); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.balance().total_pending, 0); + assert!(cold.pending_txs.contains_key(&original.txid())); + assert!(cold.pending_submissions().is_empty()); + let row = cold + .history_matching(1, None, Some(&original.txid().to_string())) + .remove(0); + assert_eq!( + (row.received_zat, row.ironwood_received, row.spent_zat), + (47000, 30000, 27000) + ); + } + + #[test] + fn confirmed_conflict_requires_all_inputs_clear_before_reactivating_saved_intent() { + use zcash_primitives::transaction::{Authorized, TransactionData}; + let (mut w, template, _) = confirmed_conflict_fixture(); + w.abandon_unmined(&template.txid().to_string()).unwrap(); + let first = w.utxos[0].clone(); + let mut second = first.clone(); + second.txid = "08".repeat(32); + second.value_zat = 10000; + w.apply_utxos_json(&serde_json::json!({"utxos":[ + {"txid":first.txid,"index":0,"script":first.script,"height":1,"valueZat":27000,"address":first.address}, + {"txid":second.txid,"index":0,"script":second.script,"height":1,"valueZat":10000,"address":second.address} + ]}).to_string()).unwrap(); + let mut bundle = template.transparent_bundle().unwrap().clone(); + bundle.vin.push(transparent::bundle::TxIn::from_parts( + transparent::bundle::OutPoint::new([8; 32], 0), + transparent::address::Script::default(), + u32::MAX, + )); + bundle.vout[0] = transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(27000).unwrap(), + bundle.vout[0].script_pubkey().clone(), + ); + let original = TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 6.into(), + Some(bundle.clone()), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &original, false); + let replacements: Vec<_> = [0usize, 1] + .into_iter() + .map(|input| { + let mut selected = bundle.clone(); + selected.vin = vec![selected.vin[input].clone()]; + selected.vout[0] = transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(if input == 0 { + 17000 + } else { + 7000 + }) + .unwrap(), + transparent::address::TransparentAddress::PublicKeyHash([5; 20]) + .script() + .into(), + ); + TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 10.into(), + Some(selected), + None, + None, + None, + ) + .freeze() + .unwrap() + }) + .collect(); + let mut one_conflict = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + include_conflicting_spender(&mut one_conflict, &replacements[0]); + assert!( + one_conflict + .utxos + .iter() + .find(|u| u.txid == "08".repeat(32)) + .unwrap() + .spent + ); + assert_eq!(one_conflict.balance().total_pending, 0); + for height in 4..=6 { + one_conflict + .apply_compact_blocks_blob(&super::super::super::encode_delimited([ + zcash_client_backend::proto::compact_formats::CompactBlock { + height, + hash: vec![height as u8; 32], + prev_hash: vec![height as u8 - 1; 32], + ..Default::default() + }, + ])) + .unwrap(); + } + let expired = WebWallet::from_snapshot(&one_conflict.to_snapshot().unwrap()).unwrap(); + assert!(!expired.pending_txs.contains_key(&original.txid())); + assert_eq!(expired.balance().total_pending, 10000); + assert!( + expired + .utxos + .iter() + .find(|u| u.txid == "09".repeat(32)) + .unwrap() + .spent + ); + assert!( + !expired + .utxos + .iter() + .find(|u| u.txid == "08".repeat(32)) + .unwrap() + .spent + ); + include_conflicting_spenders(&mut w, &[(1, &replacements[0]), (2, &replacements[1])]); + assert!(w.pending_submissions().is_empty()); + assert_eq!(w.rewind_to_height(2).unwrap(), 2); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold.pending_submissions().is_empty()); + assert_eq!( + cold.utxos + .iter() + .find(|u| u.txid == "09".repeat(32)) + .unwrap() + .spent_in + .as_deref(), + Some(super::super::super::to_hex(replacements[0].txid().as_ref()).as_str()) + ); + // The surviving conflict still blocks change, while both known inputs + // remain reserved by the retained signed intent until expiry/abandon. + assert_eq!(cold.balance().total_pending, 0); + assert_eq!(cold.rewind_to_height(1).unwrap(), 1); + let cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.pending_submissions().len(), 1); + assert_eq!(cold.balance().transparent_pending, 27000); + assert!(cold.utxos.iter().all(|u| u.spent)); + } + + #[test] + fn publisher_only_spend_does_not_disqualify_saved_independent_intent() { + let (mut w, parent) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &parent, false); + let own = external_receipt_spender(&parent, 10); + store_external_receipt_spender(&mut w, &own).unwrap(); + let other = external_receipt_spender(&parent, 11); + let mut claimed_receive = receive(); + if let TransparentEvent::Receive(event) = &mut claimed_receive { + event.txid = Txid(*parent.txid().as_ref()); + event.value = 17000; + } + let mut claimed_spend = spend(); + if let TransparentEvent::Spend(event) = &mut claimed_spend { + event.spending_txid = Txid(*other.txid().as_ref()); + event.spent_txid = Txid(*parent.txid().as_ref()); + } + let cp = checkpoint(&mut w, vec![claimed_receive, claimed_spend]); + apply_pir(&mut w, &cp); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold + .pending_submissions() + .iter() + .any(|entry| entry.txid == own.txid().to_string())); + assert!(!cold + .ineligible_pending_transactions() + .contains(&super::super::super::to_hex(own.txid().as_ref()))); + assert!( + cold.pir_funding_unverified(&super::super::super::to_hex(parent.txid().as_ref()), 0) + ); + assert_eq!(cold.balance().transparent_available, 0); + } + + #[test] + fn confirmed_conflict_propagates_through_rewound_saved_descendant_and_cold() { + use zcash_primitives::transaction::{Authorized, TransactionData}; + let (mut w, template, replacement) = confirmed_conflict_fixture(); + w.abandon_unmined(&template.txid().to_string()).unwrap(); + let parent = TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 20.into(), + template.transparent_bundle().cloned(), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &parent, false); + let child = external_receipt_spender(&parent, 21); + let blocks = public_spender_blocks(&mut w, &[(1, &parent), (2, &child)]); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks[..2].to_vec())) + .unwrap(); + assert_eq!( + w.txs()[&super::super::super::to_hex(parent.txid().as_ref())].mined_height, + Some(2) + ); + store_external_receipt_spender(&mut w, &child).unwrap(); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks[2..].to_vec())) + .unwrap(); + assert_eq!( + w.txs()[&super::super::super::to_hex(child.txid().as_ref())].mined_height, + Some(3) + ); + w.rewind_to_height(1).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.pending_submissions().len(), 2); + let new_chain = public_spender_blocks(&mut cold, &[(1, &replacement)]); + cold.apply_compact_blocks_blob(&super::super::super::encode_delimited( + new_chain[1..] + .iter() + .map(|block| { + let mut compact = block.clone(); + compact.vtx.clear(); + compact + }) + .collect::>(), + )) + .unwrap(); + cold.apply_transparent_blocks(&super::super::super::encode_delimited( + new_chain[1..].to_vec(), + )) + .unwrap(); + let mut cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + assert!(cold.pending_submissions().is_empty()); + assert!(cold.pending_txs.contains_key(&parent.txid())); + assert!(cold.pending_txs.contains_key(&child.txid())); + assert_eq!(cold.balance().total_pending, 0); + cold.rewind_to_height(1).unwrap(); + let cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.pending_submissions().len(), 2); + assert_eq!(cold.balance().total_pending, 0); + let receipt = &cold.txs()[&super::super::super::to_hex(parent.txid().as_ref())] + .accounted_transparent_receipts[&0]; + assert_eq!( + receipt.spent_in.as_ref().unwrap().txid, + super::super::super::to_hex(child.txid().as_ref()) + ); + } + + #[cfg(feature = "native")] + #[test] + fn confirmed_conflict_preserves_shielded_child_origin_after_actual_mining_rewind_and_cold() { + use zcash_client_backend::data_api::{SentTransaction, WalletWrite}; + use zcash_primitives::transaction::Transaction; + let (mut w, parent) = builder_mixed_fixture_pool(true, false); + let mut parent_raw = Vec::new(); + parent.write(&mut parent_raw).unwrap(); + let mut parent_block = w.mined_raw_tx_block(&parent_raw, 1).unwrap(); + parent_block.prev_hash = vec![3; 32]; + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([parent_block])) + .unwrap(); + assert_eq!(w.balance().orchard_available, 30000); + let destination = w.unified_address().to_owned(); + // The parent fixture has genuine encrypted notes but inert T signatures. + // This child uses the normal Rust proposal/proving/WalletWrite path. + let child_raw = w + .prove_send(REGTEST_FAUCET_MNEMONIC, &destination, "0.00001", None) + .unwrap(); + let child = + Transaction::read(&child_raw[..], zcash_protocol::consensus::BranchId::Nu6_3).unwrap(); + let parent_id = super::super::super::to_hex(parent.txid().as_ref()); + let child_id = super::super::super::to_hex(child.txid().as_ref()); + assert!(w.txs()[&child_id] + .shielded_input_origins + .0 + .as_ref() + .unwrap() + .contains(&parent_id)); + w.apply_mined_raw_tx(&child_raw, 1).unwrap(); + w.rewind_to_height(3).unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold.notes().iter().all(|note| note.txid != parent_id)); + assert!(cold.spend_notes.iter().all(|note| note.txid != parent_id)); + let known = cold.txs()[&child_id].shielded_input_origins.0.clone(); + let fee = zcash_protocol::value::Zatoshis::from_u64(cold.txs()[&child_id].fee_zat.unwrap()) + .unwrap(); + cold.store_transactions_to_be_sent(&[SentTransaction::new( + &child, + std::time::UNIX_EPOCH.into(), + 5.into(), + 0, + &[], + fee, + &[], + )]) + .unwrap(); + assert_eq!(cold.txs()[&child_id].shielded_input_origins.0, known); + let mut bundle = parent.transparent_bundle().unwrap().clone(); + bundle.vout[0] = transparent::bundle::TxOut::new( + bundle.vout[0].value(), + transparent::address::TransparentAddress::PublicKeyHash([5; 20]) + .script() + .into(), + ); + let replacement = zcash_primitives::transaction::TransactionData::< + zcash_primitives::transaction::Authorized, + >::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 10.into(), + Some(bundle), + None, + None, + None, + ) + .freeze() + .unwrap(); + include_conflicting_spender(&mut cold, &replacement); + let cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + assert!(cold.ineligible_pending_transactions().contains(&parent_id)); + assert!(cold.ineligible_pending_transactions().contains(&child_id)); + assert!(cold.pending_submissions().is_empty()); + assert!(cold.pending_txs.contains_key(&parent.txid())); + assert!(cold.pending_txs.contains_key(&child.txid())); + assert_eq!(cold.balance().total_pending, 0); + } + + #[test] + fn confirmed_conflict_unknown_old_own_lineage_refuses_without_erasing_signed_intent() { + let (w, original, _) = confirmed_conflict_fixture(); + let id = super::super::super::to_hex(original.txid().as_ref()); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][&id] + .as_object_mut() + .unwrap() + .remove("shieldedInputOrigins"); + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], true); + assert!(cold.pending_txs.contains_key(&original.txid())); + assert!(cold.pending_submissions().is_empty()); + assert_eq!(cold.balance().total_pending, 0); + let before = cold.to_snapshot().unwrap(); + assert!(builder_accounting_store_result(&mut cold, &original, false).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + assert!(cold.rescan_from(1).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + cold.pending_txs.remove(&original.txid()); + assert!( + !cold.legacy_transparent_receipt_accounting(), + "old non-signed history does not require lineage metadata" + ); + } + + fn assert_retained_intent_guard(unknown_origin: bool) { + let (mut w, original, replacement) = confirmed_conflict_fixture(); + let saved = w.pending_txs[&original.txid()].clone(); + if unknown_origin { + let mut old: serde_json::Value = + serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][super::super::super::to_hex(original.txid().as_ref())] + .as_object_mut() + .unwrap() + .remove("shieldedInputOrigins"); + w = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + } else { + include_conflicting_spender(&mut w, &replacement); + } + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert!(cold.pending_submissions().is_empty()); + assert!(cold.pending_txs[&original.txid()] == saved); + assert_eq!( + cold.pending_raw_txs().len(), + 1, + "retained intent must remain visible to pending:'reject' deletion and rescan guards" + ); + } + + #[test] + fn retained_intent_guard_keeps_known_conflict_bytes() { + assert_retained_intent_guard(false); + } + + #[test] + fn retained_intent_guard_keeps_unknown_origin_bytes() { + assert_retained_intent_guard(true); + } + + fn clear_saved_shielded_origins(w: &WebWallet, id: &str) -> WebWallet { + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][id] + .as_object_mut() + .unwrap() + .remove("shieldedInputOrigins"); + WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap() + } + + #[test] + fn reservation_restore_unknown_signed_intent_keeps_known_input_locked_after_cold_rewind() { + use zcash_client_backend::data_api::InputSource; + let (mut w, original, replacement) = confirmed_conflict_fixture(); + let id = super::super::super::to_hex(original.txid().as_ref()); + let saved = w.pending_txs[&original.txid()].clone(); + include_conflicting_spender(&mut w, &replacement); + let mut cold = clear_saved_shielded_origins(&w, &id); + cold.rewind_to_height(2).unwrap(); + assert_eq!( + cold.utxos[0].spent_in.as_deref(), + Some(super::super::super::to_hex(replacement.txid().as_ref()).as_str()), + "a different still-current owner must win" + ); + cold.rewind_to_height(1).unwrap(); + let mut cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + let input = transparent::bundle::OutPoint::new([9; 32], 0); + assert!( + cold.get_unspent_transparent_output(&input, 100.into()) + .unwrap() + .is_none(), + "unknown lineage must not release its exact known transparent reservation" + ); + assert_eq!(cold.utxos[0].spent_in.as_deref(), Some(id.as_str())); + assert!(cold.pending_submissions().is_empty()); + assert_eq!(cold.pending_raw_txs().len(), 1); + assert!(cold.pending_txs[&original.txid()] == saved); + assert_eq!(cold.balance().total_pending, 0); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], true); + let before = cold.to_snapshot().unwrap(); + assert!(builder_accounting_store_result(&mut cold, &original, false).is_err()); + assert!(cold.rescan_from(1).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + let mut abandoned = WebWallet::from_snapshot(&before).unwrap(); + assert!(abandoned + .abandon_unmined(&original.txid().to_string()) + .unwrap()); + assert!(!abandoned.utxos[0].spent); + assert!(abandoned.utxos[0].spent_in.is_none()); + } + + #[test] + fn reservation_restore_unknown_ancestor_keeps_saved_child_claim_without_creating_funds() { + use zcash_primitives::transaction::{Authorized, TransactionData}; + let (mut w, template, _) = confirmed_conflict_fixture(); + w.abandon_unmined(&template.txid().to_string()).unwrap(); + let parent = TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 20.into(), + template.transparent_bundle().cloned(), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &parent, false); + let child = external_receipt_spender(&parent, 21); + let blocks = public_spender_blocks(&mut w, &[(1, &parent), (2, &child)]); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks[..2].to_vec())) + .unwrap(); + store_external_receipt_spender(&mut w, &child).unwrap(); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks[2..].to_vec())) + .unwrap(); + // The parent remains mined while an independent spender replaces its child. + w.rewind_to_height(2).unwrap(); + let other = external_receipt_spender(&parent, 22); + let new_chain = public_spender_blocks(&mut w, &[(1, &parent), (2, &other)]); + w.apply_compact_blocks_blob(&super::super::super::encode_delimited( + new_chain[2..] + .iter() + .map(|b| { + let mut b = b.clone(); + b.vtx.clear(); + b + }) + .collect::>(), + )) + .unwrap(); + w.apply_transparent_blocks(&super::super::super::encode_delimited( + new_chain[2..].to_vec(), + )) + .unwrap(); + let parent_id = super::super::super::to_hex(parent.txid().as_ref()); + let child_id = super::super::super::to_hex(child.txid().as_ref()); + assert_eq!(w.txs()[&parent_id].mined_height, Some(2)); + assert_eq!( + w.txs()[&parent_id].accounted_transparent_receipts[&0] + .spent_in + .as_ref() + .unwrap() + .txid, + super::super::super::to_hex(other.txid().as_ref()) + ); + let mut cold = clear_saved_shielded_origins(&w, &parent_id); + cold.rewind_to_height(1).unwrap(); + let cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + let receipt = &cold.txs()[&parent_id].accounted_transparent_receipts[&0]; + assert_eq!( + receipt.spent_in.as_ref().map(|spent| spent.txid.as_str()), + Some(child_id.as_str()), + "known child reservation survives unknown ancestor eligibility" + ); + assert_eq!(cold.utxos[0].spent_in.as_deref(), Some(parent_id.as_str())); + assert!(cold.pending_submissions().is_empty()); + assert_eq!(cold.pending_raw_txs().len(), 2); + assert_eq!(cold.balance().total_pending, 0); + assert!( + !cold.utxos.iter().any(|u| u.txid == parent_id), + "restoration never creates missing funding rows" + ); + } + + #[test] + fn expired_debit_is_history_not_an_extra_current_balance() { + use zcash_primitives::transaction::{Authorized, TransactionData}; + let (mut w, template) = builder_accounting_fixture(); + let tx = TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + 4.into(), + template.transparent_bundle().cloned(), + None, + None, + None, + ) + .freeze() + .unwrap(); + builder_accounting_store(&mut w, &tx, false); + let before = w + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([ + zcash_client_backend::proto::compact_formats::CompactBlock { + height: 4, + hash: vec![4; 32], + prev_hash: vec![3; 32], + ..Default::default() + }, + ])) + .unwrap(); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let expired = cold + .history_matching(1, None, Some(&tx.txid().to_string())) + .remove(0); + assert!(expired.expired_unmined); + assert_eq!( + ( + expired.spent_zat, + expired.sent_note_count, + expired.account_delta_zat + ), + ( + before.spent_zat, + before.sent_note_count, + before.account_delta_zat + ) + ); + assert!(cold.pending_submissions().is_empty()); + assert_eq!(cold.balance().total_pending, 27000); + assert!(cold.pending_raw_txs().is_empty()); + } + + fn external_receipt_spender( + parent: &zcash_primitives::transaction::Transaction, + expiry: u32, + ) -> zcash_primitives::transaction::Transaction { + use zcash_primitives::transaction::{Authorized, TransactionData}; + TransactionData::::from_parts_v6( + zcash_protocol::consensus::BranchId::Nu6_3, + 0, + expiry.into(), + Some(transparent::bundle::Bundle { + vin: vec![transparent::bundle::TxIn::from_parts( + transparent::bundle::OutPoint::new(*parent.txid().as_ref(), 0), + transparent::address::Script::default(), + u32::MAX, + )], + vout: vec![transparent::bundle::TxOut::new( + zcash_protocol::value::Zatoshis::from_u64(7000).unwrap(), + transparent::address::TransparentAddress::PublicKeyHash([5; 20]) + .script() + .into(), + )], + authorization: transparent::bundle::Authorized, + }), + None, + None, + None, + ) + .freeze() + .unwrap() + } + + fn store_external_receipt_spender( + w: &mut WebWallet, + tx: &zcash_primitives::transaction::Transaction, + ) -> std::result::Result<(), String> { + use zcash_address::ToAddress; + use zcash_client_backend::data_api::{SentTransaction, SentTransactionOutput, WalletWrite}; + use zcash_client_backend::wallet::Recipient; + let outputs = [SentTransactionOutput::from_parts( + 0, + Recipient::External { + recipient_address: zcash_address::ZcashAddress::from_transparent_p2pkh( + zcash_protocol::consensus::NetworkType::Regtest, + [5; 20], + ), + output_pool: zcash_protocol::PoolType::Transparent, + }, + zcash_protocol::value::Zatoshis::from_u64(7000).unwrap(), + None, + )]; + w.store_transactions_to_be_sent(&[SentTransaction::new( + tx, + std::time::UNIX_EPOCH.into(), + 3.into(), + 0, + &outputs, + zcash_protocol::value::Zatoshis::from_u64(10000).unwrap(), + &[], + )]) + } + + #[test] + fn durable_debit_public_receipt_only_spend_survives_expiry_rewind_and_replay() { + let (mut w, parent) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &parent, true); + let child = external_receipt_spender(&parent, 4); + for _ in 0..2 { + store_external_receipt_spender(&mut w, &child).unwrap(); + } + assert_eq!(w.balance().transparent_pending, 0); + let row = w + .history_matching(1, None, Some(&child.txid().to_string())) + .remove(0); + assert_eq!( + ( + row.spent_zat, + row.sent_note_count, + row.transparent_spent, + row.account_delta_zat + ), + (17000, 1, 17000, -17000) + ); + let other = external_receipt_spender(&parent, 5); + let before = w.to_snapshot().unwrap(); + assert!(store_external_receipt_spender(&mut w, &other).is_err()); + assert!( + w.to_snapshot().unwrap() == before, + "conflicting callback must not save another signed intent" + ); + assert!(w.abandon_unmined(&child.txid().to_string()).unwrap()); + assert_eq!(w.balance().transparent_pending, 17000); + for _ in 0..2 { + store_external_receipt_spender(&mut w, &child).unwrap(); + } + let row = w + .history_matching(1, None, Some(&child.txid().to_string())) + .remove(0); + assert_eq!( + (row.spent_zat, row.sent_note_count, row.transparent_spent), + (17000, 1, 17000) + ); + let mut blocks = public_range(&mut w, false); + blocks[0].vtx.pop(); + blocks[1] + .vtx + .push(zcash_client_backend::proto::compact_formats::CompactTx { + index: 1, + txid: child.txid().as_ref().to_vec(), + vin: vec![zcash_client_backend::proto::compact_formats::CompactTxIn { + prevout_txid: parent.txid().as_ref().to_vec(), + prevout_index: 0, + }], + vout: vec![zcash_client_backend::proto::compact_formats::TxOut { + value: 7000, + script_pub_key: child.transparent_bundle().unwrap().vout[0] + .script_pubkey() + .0 + .0 + .clone(), + }], + ..Default::default() + }); + w.apply_transparent_blocks(&super::super::super::encode_delimited(blocks)) + .unwrap(); + store_external_receipt_spender(&mut w, &child).unwrap(); + let id = super::super::super::to_hex(child.txid().as_ref()); + assert_eq!(w.txs().get(&id).unwrap().mined_height, Some(2)); + w.apply_compact_blocks_blob(&super::super::super::encode_delimited([ + zcash_client_backend::proto::compact_formats::CompactBlock { + height: 4, + hash: vec![4; 32], + prev_hash: vec![3; 32], + ..Default::default() + }, + ])) + .unwrap(); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + let previous = super::super::super::to_hex(parent.txid().as_ref()); + assert_eq!( + cold.txs() + .get(&previous) + .unwrap() + .accounted_transparent_receipts[&0] + .spent_in + .as_ref() + .unwrap() + .height, + Some(2) + ); + assert_eq!(cold.balance().transparent_pending, 0); + assert_eq!(cold.txs().get(&id).unwrap().mined_height, Some(2)); + let row = cold + .history_matching(1, None, Some(&child.txid().to_string())) + .remove(0); + assert_eq!( + (row.spent_zat, row.sent_note_count, row.transparent_spent), + (17000, 1, 17000) + ); + assert!(!cold + .pending_submissions() + .iter() + .any(|pending| pending.txid == child.txid().to_string())); + let before = cold.to_snapshot().unwrap(); + assert!(cold.abandon_unmined(&child.txid().to_string()).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + cold.rewind_to_height(0).unwrap(); + cold = WebWallet::from_snapshot(&cold.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.txs().get(&id).unwrap().mined_height, None); + assert!(cold.pending_txs.contains_key(&child.txid())); + assert_eq!(cold.balance().transparent_pending, 0); + for height in 1u32..=4 { + cold.apply_compact_blocks_blob(&super::super::super::encode_delimited([ + zcash_client_backend::proto::compact_formats::CompactBlock { + height: height.into(), + hash: vec![(height * 11) as u8; 32], + prev_hash: if height == 1 { + vec![0; 32] + } else { + vec![((height - 1) * 11) as u8; 32] + }, + ..Default::default() + }, + ])) + .unwrap(); + } + assert_eq!( + cold.balance().transparent_pending, + 17000, + "only genuinely unmined expiry releases the parent claim" + ); + } + + #[test] + fn durable_debit_old_pointer_without_credit_identity_refuses_before_mutation() { + let (mut w, parent) = builder_accounting_fixture(); + builder_accounting_store(&mut w, &parent, false); + let child = external_receipt_spender(&parent, 20); + store_external_receipt_spender(&mut w, &child).unwrap(); + let previous = super::super::super::to_hex(parent.txid().as_ref()); + let id = super::super::super::to_hex(child.txid().as_ref()); + let mut old: serde_json::Value = serde_json::from_slice(&w.to_snapshot().unwrap()).unwrap(); + old["txs"][&previous]["accountedTransparentReceipts"]["0"]["spentIn"] + .as_object_mut() + .unwrap() + .remove("accounted"); + for (_, agg) in old["txs"].as_object_mut().unwrap() { + agg.as_object_mut() + .unwrap() + .remove("accountedTransparentSpends"); + } + let mut cold = WebWallet::from_snapshot(&serde_json::to_vec(&old).unwrap()).unwrap(); + assert_eq!(cold.transparent_pir_status()["rescanRequired"], true); + let before = cold.to_snapshot().unwrap(); + assert!(store_external_receipt_spender(&mut cold, &child).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + assert!(cold.rescan_from(1).is_err()); + assert!(cold.to_snapshot().unwrap() == before); + assert!(cold.pending_txs.contains_key(&child.txid())); + assert!(cold.txs().contains_key(&id)); + } + + fn off_refresh_fixture() -> (WebWallet, TransparentPirCheckpoint) { + let mut w = wallet(); + w.next_unified_address().unwrap(); + let current = w.transparent_address().unwrap().to_owned(); + let script = w + .public_transparent_scripts() + .unwrap() + .into_iter() + .find(|(_, address)| address == ¤t) + .unwrap() + .0; + let mut c = checkpoint(&mut w, vec![receive()]); + let mut current_receive = receive(); + if let TransparentEvent::Receive(event) = &mut current_receive { + event.txid = Txid([3; 32]); + event.height = 3; + event.value = 17000; + event.coinbase = true; + event.transaction_index = 0; + } + c.records.push(TransparentPirRecord { + script: super::super::super::to_hex(&script), + event: super::super::super::to_hex(¤t_receive.to_bytes()), + shard_id: 0, + revision: "ef".repeat(32), + }); + c.continuation = "opaque-acquisition-with-logical-cover-debt".into(); + apply_pir(&mut w, &c); + assert_ne!( + w.utxos + .iter() + .find(|u| u.txid == "01".repeat(32)) + .unwrap() + .address, + current + ); + (w, c) + } + fn off_refresh_listed(w: &WebWallet, byte: u8) -> serde_json::Value { + let id = format!("{byte:02x}").repeat(32); + let u = w.utxos.iter().find(|u| u.txid == id).unwrap(); + serde_json::json!({"txid":u.txid,"index":u.index,"script":u.script, + "valueZat":u.value_zat,"height":u.height,"address":u.address,"coinbase":false}) + } + #[test] + fn off_refresh_retains_unverified_pir_facts_on_all_issued_addresses_and_cold() { + let (mut w, _) = off_refresh_fixture(); + let acquisition = w.transparent_pir_checkpoint(); + assert_eq!(w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(), 0); + assert_eq!(w.utxos.len(), 2); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "167000"); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 167000); + assert_eq!(w.transparent_pir_checkpoint(), acquisition); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.utxos.len(), 2); + assert_eq!(cold.balance().transparent_pending, 167000); + assert_eq!(cold.transparent_pir_checkpoint(), acquisition); + let before = cold.to_snapshot().unwrap(); + assert!(cold + .begin_transparent_pir(&"ac".repeat(32)) + .unwrap_err() + .to_string() + .contains("transparent_pir_source_changed")); + assert_eq!(cold.to_snapshot().unwrap(), before); + } + #[test] + fn off_refresh_preserves_pir_reservations_and_allows_only_authoritative_retirement() { + use zcash_client_backend::{ + data_api::OutputLockStore, + wallet::{LockOwner, OutputRef}, + }; + let (mut w, c) = off_refresh_fixture(); + let owner = LockOwner::new([9; 32]); + let output = OutputRef::new( + zcash_primitives::transaction::TxId::from_bytes([1; 32]), + zcash_protocol::PoolType::TRANSPARENT, + 0, + ); + w.lock_outputs(&[output.clone()], owner, 100.into()) + .unwrap(); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_eq!(w.utxos.len(), 2); + // OutputLockStore is intentionally session-local; the durable signed + // submission fixture separately covers cold unresolved-payment ownership. + assert_eq!( + w.active_lock_owner(&"01".repeat(32), "transparent", 0), + Some("09".repeat(32).as_str()) + ); + let mut retirement = checkpoint(&mut w, vec![]); + retirement.checkpoint_id = "de".repeat(32); + retirement.previous_checkpoint_id = Some(c.checkpoint_id); + retirement.rewind_to = Some(TransparentPirAnchor { + height: 0, + hash: Network::Regtest.genesis_hash().into(), + }); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&retirement).unwrap()) + .unwrap_err() + .to_string() + .contains("transparent_pir_pending_reconciliation_required")); + assert_eq!(w.to_snapshot().unwrap(), before); + assert!(w.unlock_output(&output, owner).unwrap()); + let mut cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + apply_pir(&mut cold, &retirement); + assert!(cold.utxos.is_empty()); + assert_eq!(cold.balance().transparent_pending, 0); + assert_eq!(cold.transparent_pir_status()["confirmedZat"], "0"); + } + #[test] + fn off_refresh_preserves_ordinary_replacement_listed_pir_coinbase_and_atomic_errors() { + let (mut w, _) = off_refresh_fixture(); + let mut ordinary = off_refresh_listed(&w, 3); + ordinary["txid"] = serde_json::json!("07".repeat(32)); + ordinary["valueZat"] = serde_json::json!(700); + w.apply_utxos_json(&serde_json::json!({"utxos":[ordinary]}).to_string()) + .unwrap(); + assert_eq!(w.utxos.len(), 3); + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_eq!(w.utxos.len(), 2); + assert!(!w.utxos.iter().any(|u| u.txid == "07".repeat(32))); + let row = off_refresh_listed(&w, 3); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_utxos_json(&serde_json::json!({"utxos":[row.clone(),{"index":0}]}).to_string()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + w.apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(w.utxos.len(), 2); + assert!(!w.pir_funding_unverified(&"03".repeat(32), 0)); + assert!(w.pir_funding_unverified(&"01".repeat(32), 0)); + assert_eq!( + w.utxos + .iter() + .find(|u| u.txid == "03".repeat(32)) + .unwrap() + .coinbase, + CoinbaseStatus::Coinbase + ); + assert_eq!(w.balance().transparent_available, 0); + w.scanned_height = 105; + assert_eq!(w.balance().transparent_available, 17000); + assert_eq!(w.balance().transparent_pending, 150000); + // Independently promoted rows use ordinary replacement semantics thereafter. + w.apply_utxos_json(r#"{"utxos":[]}"#).unwrap(); + assert_eq!(w.utxos.len(), 1); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + } + + #[test] + fn receive_then_spend_offline_projects_zero_and_real_history() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![spend(), receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "0"); + assert_eq!(w.history(20).len(), 2); + assert_eq!(w.txs[&"01".repeat(32)].transparent_received, 150000); + assert_eq!(w.txs[&"02".repeat(32)].transparent_spent, 150000); + let saved = w.to_snapshot().unwrap(); + let restored = WebWallet::from_snapshot(&saved).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "0"); + assert_eq!(restored.history(20).len(), 2); + } + #[test] + fn transparent_pir_rejects_impossible_publisher_amount_atomically() { + let mut w = wallet(); + let mut event = receive(); + if let TransparentEvent::Receive(e) = &mut event { + e.value = u64::MAX; + } + let c = checkpoint(&mut w, vec![event]); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert!(w.utxos.is_empty()); + } + #[test] + fn late_conflict_rolls_back_and_old_generation_cannot_apply() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + let before = w.to_snapshot().unwrap(); + let mut bad = c.clone(); + bad.records.push(TransparentPirRecord { + script: "00".into(), + event: "00".into(), + shard_id: 0, + revision: "ef".repeat(32), + }); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&bad).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert!(w.begin_transparent_pir(&"ff".repeat(32)).is_err()); + w.scanned_height = 4; + w.block_hashes.insert(4, "04".repeat(32)); + w.begin_transparent_pir(&"ab".repeat(32)).unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .is_err()); + assert!(w.utxos.is_empty()); + } + #[test] + fn known_coinbase_zero_conf_never_becomes_available() { + let mut w = wallet(); + let script = w.public_transparent_scripts().unwrap()[0].0.clone(); + let row = serde_json::json!({ + "txid":"01".repeat(32),"index":0,"script":super::super::super::to_hex(&script), + "valueZat":150000,"height":0,"coinbase":true + }); + w.apply_utxos_json(&serde_json::json!({"utxos":[row]}).to_string()) + .unwrap(); + assert_eq!(w.utxos[0].coinbase, CoinbaseStatus::Coinbase); + assert_eq!(w.balance().transparent_available, 0); + assert_eq!(w.balance().transparent_pending, 150000); + assert!(w.prove_shield(REGTEST_FAUCET_MNEMONIC, 1).is_err()); + assert!(w.pending_txs.is_empty()); + let cold = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(cold.utxos[0].coinbase, CoinbaseStatus::Coinbase); + assert_eq!(cold.balance().transparent_available, 0); + assert_eq!(cold.balance().transparent_pending, 150000); + w.scanned_height = 1000; + assert_eq!(w.balance().transparent_available, 0); + } +} + +#[cfg(test)] +mod retirement_tests { + use super::tests::*; + use super::*; + #[test] + fn transparent_pir_provisional_replacement_retires_only_projected_spend() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + let mut next = checkpoint(&mut w, vec![receive()]); + next.checkpoint_id = "de".repeat(32); + next.previous_checkpoint_id = Some(c.checkpoint_id); + next.rewind_to = Some(TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&next).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&next.checkpoint_id) + .unwrap(); + assert_eq!(w.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(w.history(20).len(), 1); + let restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn transparent_pir_canonical_rewind_can_resume_partial_and_restore_received_balance() { + let mut w = wallet(); + w.block_hashes.insert(1, "01".repeat(32)); + let c = checkpoint(&mut w, vec![receive(), spend()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.acknowledge_transparent_pir_checkpoint(&c.checkpoint_id) + .unwrap(); + let saved = w.transparent_pir_checkpoint(); + w.rewind_to_height(1).unwrap(); + assert!( + w.transparent_pir_checkpoint() == saved, + "canonical rewind retains private acquisition" + ); + assert!( + w.begin_transparent_pir(&"ab".repeat(32)).is_err(), + "lower target remains closed until scanning catches up" + ); + let compact = crate::web::encode_delimited((2..=3).map(|height| { + zcash_client_backend::proto::compact_formats::CompactBlock { + height, + hash: vec![height as u8 + 0x80; 32], + prev_hash: vec![if height == 2 { 1 } else { 0x82 }; 32], + time: 1, + chain_metadata: Some(Default::default()), + ..Default::default() + } + })); + w.apply_compact_blocks_blob(&compact).unwrap(); + let mut partial = checkpoint(&mut w, vec![]); + partial.checkpoint_id = "de".repeat(32); + partial.previous_checkpoint_id = Some(c.checkpoint_id); + partial.rewind_to = Some(TransparentPirAnchor { + height: 1, + hash: "01".repeat(32), + }); + partial.covered_through = 1; + for c in &mut partial.coverage { + c.through = 1; + } + partial.complete = false; + partial.pending_pages = 1; + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&partial).unwrap()) + .unwrap(); + let mut restored = WebWallet::from_snapshot(&w.to_snapshot().unwrap()).unwrap(); + assert_eq!( + restored.transparent_pir_status()["completion"], + "incomplete" + ); + let mut complete = checkpoint(&mut restored, vec![receive()]); + complete.checkpoint_id = "ef".repeat(32); + complete.previous_checkpoint_id = Some(partial.checkpoint_id); + restored + .apply_transparent_pir_checkpoint(&serde_json::to_string(&complete).unwrap()) + .unwrap(); + restored + .acknowledge_transparent_pir_checkpoint(&complete.checkpoint_id) + .unwrap(); + assert_eq!(restored.transparent_pir_status()["confirmedZat"], "150000"); + assert_eq!(restored.history(20).len(), 1); + } + #[test] + fn transparent_pir_retirement_cannot_remove_reserved_output_and_chain_rewind_invalidates() { + let mut w = wallet(); + let c = checkpoint(&mut w, vec![receive()]); + w.apply_transparent_pir_checkpoint(&serde_json::to_string(&c).unwrap()) + .unwrap(); + w.output_locks.push(StoredLock { + txid: "01".repeat(32), + pool: "transparent".into(), + index: 0, + owner: "ff".repeat(32), + expiry: 100, + }); + let mut next = checkpoint(&mut w, vec![]); + next.checkpoint_id = "de".repeat(32); + next.previous_checkpoint_id = Some(c.checkpoint_id); + next.rewind_to = Some(TransparentPirAnchor { + height: 0, + hash: Network::Regtest.genesis_hash().into(), + }); + let before = w.to_snapshot().unwrap(); + assert!(w + .apply_transparent_pir_checkpoint(&serde_json::to_string(&next).unwrap()) + .is_err()); + assert_eq!(w.to_snapshot().unwrap(), before); + assert_eq!(w.output_locks.len(), 1); + w.block_hashes.insert(1, "01".repeat(32)); + w.rewind_to_height(1).unwrap(); + assert_ne!( + w.transparent_pir_status()["completion"], + "complete-for-enrolled-scope" + ); + } +} diff --git a/crates/z-engine/src/web/write.rs b/crates/z-engine/src/web/write.rs index 2655935..1bcf6ec 100644 --- a/crates/z-engine/src/web/write.rs +++ b/crates/z-engine/src/web/write.rs @@ -406,12 +406,18 @@ impl InputSource for WebWallet { target_height: TargetHeight, ) -> Result>, Self::Error> { let hx = super::to_hex(outpoint.hash()); + let conflicts = self.ineligible_pending_transactions(); Ok(self .utxos .iter() .find(|u| !u.spent && u.txid == hx && u.index == outpoint.n()) .filter(|u| { - self.utxo_confirms(u, target_height, crate::confirmations_policy(self.network)) + self.utxo_confirms( + u, + target_height, + crate::confirmations_policy(self.network), + &conflicts, + ) }) .and_then(|u| self.stored_to_wto(u))) } @@ -427,13 +433,14 @@ impl InputSource for WebWallet { ) -> Result>, Self::Error> { use zcash_keys::encoding::AddressCodec; let want = address.encode(&self.network); + let conflicts = self.ineligible_pending_transactions(); Ok(self .utxos .iter() .filter(|u| { !u.spent && (u.address.is_empty() || u.address == want) - && self.utxo_confirms(u, target_height, confirmations_policy) + && self.utxo_confirms(u, target_height, confirmations_policy, &conflicts) }) .filter_map(|u| self.stored_to_wto(u)) .collect()) @@ -498,11 +505,12 @@ impl WebWallet { u: &super::store::StoredUtxo, target: TargetHeight, policy: ConfirmationsPolicy, + conflicts: &std::collections::HashSet, ) -> bool { if u.spent { return false; } - self.utxo_ready(u, target, &policy) + self.utxo_ready(u, target, &policy, conflicts) } #[cfg(feature = "transparent-inputs")] @@ -934,11 +942,12 @@ impl WalletRead for WebWallet { return Ok(HashMap::new()); } let mut map = HashMap::new(); + let conflicts = self.ineligible_pending_transactions(); for addr in self.known_taddrs() { let want = addr.encode(&self.network); let mut bal = zcash_client_backend::data_api::Balance::ZERO; for u in &self.utxos { - if u.spent { + if u.spent || conflicts.contains(&u.txid) { continue; } if !u.address.is_empty() && u.address != want { @@ -948,7 +957,7 @@ impl WalletRead for WebWallet { continue; } let v = Zatoshis::from_u64(u.value_zat).unwrap_or(Zatoshis::ZERO); - if self.utxo_confirms(u, target_height, confirmations_policy) { + if self.utxo_confirms(u, target_height, confirmations_policy, &conflicts) { let _ = bal.add_spendable_value(v); } else { let _ = bal.add_pending_spendable_value(v); @@ -1264,20 +1273,51 @@ impl WalletWrite for WebWallet { ) -> Result { let txid = super::to_hex(output.outpoint().hash()); let index = output.outpoint().n(); + #[cfg(feature = "transparent-pir")] + let funding_unverified = self.pir_funding_unverified(&txid, index); + #[cfg(not(feature = "transparent-pir"))] + let funding_unverified = false; + let accepts = self + .utxos + .iter() + .find(|u| u.txid == txid && u.index == index) + .is_none_or(|u| !u.spent && (!funding_unverified || output.mined_height().is_some())); + if accepts { + self.preflight_transparent_receipts(&[( + txid.clone(), + index, + u64::from(output.txout().value()), + )]) + .map_err(|error| error.to_string())?; + } if let Some((i, existing)) = self .utxos .iter_mut() .enumerate() .find(|(_, u)| u.txid == txid && u.index == index) { - if !existing.spent { + // An unmined write cannot replace facts whose independent funding + // verification is still pending, even after a public prefix sighting. + if !existing.spent && (!funding_unverified || output.mined_height().is_some()) { existing.height = output .mined_height() .map(u32::from) .unwrap_or(existing.height); existing.value_zat = u64::from(output.txout().value()); existing.script = super::to_hex(&output.txout().script_pubkey().0 .0); + self.credit_transparent_receipt( + &txid, + index, + u64::from(output.txout().value()), + output.mined_height().map(u32::from), + false, + ); + #[cfg(feature = "transparent-pir")] + if output.mined_height().is_some() { + self.confirm_public_transparent_output(&txid, index); + } } + self.recompute_pool_fields(); return Ok(i as u32); } #[cfg(feature = "transparent-inputs")] @@ -1287,9 +1327,13 @@ impl WalletWrite for WebWallet { }; #[cfg(not(feature = "transparent-inputs"))] let address = String::new(); + #[cfg(feature = "transparent-pir")] + if output.mined_height().is_some() { + self.confirm_public_transparent_output(&txid, index); + } self.utxos.push(super::store::StoredUtxo { - coinbase: false, - txid, + coinbase: super::store::CoinbaseStatus::Unknown, + txid: txid.clone(), index, script: super::to_hex(&output.txout().script_pubkey().0 .0), value_zat: u64::from(output.txout().value()), @@ -1298,6 +1342,14 @@ impl WalletWrite for WebWallet { spent: false, spent_in: None, }); + self.credit_transparent_receipt( + &txid, + index, + u64::from(output.txout().value()), + output.mined_height().map(u32::from), + false, + ); + self.recompute_pool_fields(); Ok(self.utxos.len() as u32 - 1) } @@ -1317,7 +1369,118 @@ impl WalletWrite for WebWallet { transactions: &[SentTransaction], ) -> Result<(), String> { use zcash_client_backend::wallet::Recipient; + #[cfg(feature = "transparent-inputs")] + let mut receipts = Vec::new(); + #[cfg(not(feature = "transparent-inputs"))] + let receipts = Vec::new(); + #[cfg(feature = "transparent-inputs")] for sent in transactions { + for output in sent.outputs() { + if matches!( + output.recipient(), + Recipient::InternalTransparent { .. } | Recipient::EphemeralTransparent { .. } + ) { + receipts.push(( + super::to_hex(sent.tx().txid().as_ref()), + output.output_index() as u32, + u64::from(output.value()), + )); + } + } + } + self.preflight_transparent_receipts(&receipts) + .map_err(|error| error.to_string())?; + let spends: Vec<_> = transactions + .iter() + .flat_map(|sent| { + sent.tx() + .transparent_bundle() + .into_iter() + .flat_map(move |bundle| { + bundle.vin.iter().map(move |input| { + ( + super::to_hex(input.prevout().hash()), + input.prevout().n(), + super::to_hex(sent.tx().txid().as_ref()), + None, + ) + }) + }) + }) + .collect(); + self.preflight_transparent_spends(&spends) + .map_err(|error| error.to_string())?; + let ineligible = self.ineligible_pending_transactions(); + let origins: Result, String> = transactions + .iter() + .map(|sent| { + let mut nullifiers = HashMap::<&str, HashSet>::new(); + for (pool, bundle) in [ + ("orchard", sent.tx().orchard_bundle()), + ("ironwood", sent.tx().ironwood_bundle()), + ] { + if let Some(bundle) = bundle { + nullifiers.entry(pool).or_default().extend( + bundle + .actions() + .iter() + .map(|action| super::to_hex(&action.nullifier().to_bytes())), + ); + } + } + if let Some(bundle) = sent.tx().sapling_bundle() { + nullifiers.entry("sapling").or_default().extend( + bundle + .shielded_spends() + .iter() + .map(|spend| super::to_hex(&spend.nullifier().0)), + ); + } + let owned_input = |pool: &str, nf: &str| { + nullifiers.get(pool).is_some_and(|known| known.contains(nf)) + }; + let observed: std::collections::BTreeSet<_> = self + .notes() + .iter() + .filter(|note| owned_input(¬e.pool, ¬e.nf)) + .map(|note| note.txid.clone()) + .chain( + self.spend_notes + .iter() + .filter(|note| owned_input(¬e.pool, ¬e.nf)) + .map(|note| note.txid.clone()), + ) + .collect(); + if self.pending_txs.contains_key(&sent.tx().txid()) + && self + .txs() + .get(&super::to_hex(sent.tx().txid().as_ref())) + .is_none_or(|agg| agg.shielded_input_origins.0.is_none()) + { + return Err("transparent_pir_rescan_required".into()); + } + if ineligible.contains(&super::to_hex(sent.tx().txid().as_ref())) + || observed.iter().any(|parent| ineligible.contains(parent)) + { + return Err("transparent_receipt_conflict".into()); + } + if self.pending_txs.contains_key(&sent.tx().txid()) { + let known = self + .txs() + .get(&super::to_hex(sent.tx().txid().as_ref())) + .and_then(|agg| agg.shielded_input_origins.0.as_ref()) + .ok_or_else(|| "transparent_pir_rescan_required".to_owned())?; + if !observed.is_subset(known) { + return Err("transparent_receipt_conflict".into()); + } + Ok(known.clone()) + } else { + Ok(observed) + } + }) + .collect(); + let origins = origins?; + for (sent, origins) in transactions.iter().zip(origins) { self.store_pending_tx(sent.tx()) .map_err(|e| e.to_string())?; let txid = super::to_hex(sent.tx().txid().as_ref()); @@ -1362,44 +1525,76 @@ impl WalletWrite for WebWallet { let prev = vin.prevout(); let hx = super::to_hex(prev.hash()); let idx = prev.n(); + let mut new_input = None; for u in &mut self.utxos { if !u.spent && u.txid == hx && u.index == idx { u.spent = true; u.spent_in = Some(txid.clone()); - spent_t = spent_t.saturating_add(u.value_zat); - spent_zat = spent_zat.saturating_add(u.value_zat); - spent_notes += 1; + new_input = Some(u.value_zat); + } + } + if let Some(value) = new_input { + self.credit_transparent_debit(&txid, &hx, idx, value); + spent_t = spent_t.saturating_add(value); + } + if let Some(value) = + self.record_transparent_receipt_spend(&hx, idx, &txid, None) + { + if new_input.is_none() { + spent_t = spent_t.saturating_add(value); } } + #[cfg(feature = "transparent-pir")] + self.record_public_transparent_spend(&txid, &hx, idx); } } + let complete_receipts = self + .txs() + .get(&txid) + .is_some_and(|agg| agg.receipt_totals_recorded); + let known_notes: HashSet<_> = self + .notes() + .iter() + .filter(|note| note.txid == txid) + .map(|note| (note.pool.clone(), note.output_index)) + .collect(); let mut received_zat = 0u64; let mut received_notes = 0u32; + #[cfg(feature = "transparent-inputs")] + let mut transparent_receipts = std::collections::BTreeMap::::new(); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_receipts = std::collections::BTreeMap::::new(); let mut ironwood_received = 0u64; let mut has_change = false; let mut memos = Vec::new(); for o in sent.outputs() { match o.recipient() { Recipient::InternalShielded { note, .. } => { - received_zat = received_zat.saturating_add(u64::from(o.value())); - received_notes += 1; has_change = true; - if matches!( - note.as_ref(), - zcash_client_backend::wallet::Note::Orchard { + let pool = match note.as_ref() { + Note::Sapling(_) => "sapling", + Note::Orchard { pool: orchard::ValuePool::Ironwood, .. + } => "ironwood", + Note::Orchard { .. } => "orchard", + }; + if !complete_receipts + && !known_notes.contains(&(pool.to_string(), o.output_index() as u32)) + { + received_zat = received_zat.saturating_add(u64::from(o.value())); + received_notes += 1; + if pool == "ironwood" { + ironwood_received = + ironwood_received.saturating_add(u64::from(o.value())); } - ) { - ironwood_received = - ironwood_received.saturating_add(u64::from(o.value())); } } #[cfg(feature = "transparent-inputs")] Recipient::InternalTransparent { .. } | Recipient::EphemeralTransparent { .. } => { - received_zat = received_zat.saturating_add(u64::from(o.value())); - received_notes += 1; + let value = u64::from(o.value()); + transparent_receipts.insert(o.output_index() as u32, value); } Recipient::External { .. } => {} } @@ -1409,14 +1604,20 @@ impl WalletWrite for WebWallet { } } } + let orchard_received = received_zat.saturating_sub(ironwood_received); + let receipt_totals_recorded = has_change || !transparent_receipts.is_empty(); + for (index, value) in transparent_receipts { + self.credit_transparent_receipt(&txid, index, value, None, true); + } let agg = self.txs_mut().entry(txid.clone()).or_default(); - agg.mined_height = None; + agg.shielded_input_origins.0 = Some(origins); + agg.receipt_totals_recorded |= receipt_totals_recorded; agg.spent_zat = agg.spent_zat.saturating_add(spent_zat); agg.spent_notes = agg.spent_notes.saturating_add(spent_notes); agg.received_zat = agg.received_zat.saturating_add(received_zat); agg.received_notes = agg.received_notes.saturating_add(received_notes); agg.has_change = agg.has_change || has_change; - agg.is_shielding = agg.is_shielding || (spent_t > 0 && received_zat > 0); + agg.is_shielding = agg.is_shielding || (spent_t > 0 && receipt_totals_recorded); let builder_fee = u64::from(sent.fee_amount()); // The total builder fee is useful history metadata, but a pool-local balance // identity additionally requires pure-Ironwood bytes and exact agreement. @@ -1429,15 +1630,18 @@ impl WalletWrite for WebWallet { agg.orchard_spent = agg.orchard_spent.saturating_add(orchard_spent); agg.ironwood_spent = agg.ironwood_spent.saturating_add(ironwood_spent); agg.ironwood_received = agg.ironwood_received.saturating_add(ironwood_received); - agg.orchard_received = agg - .orchard_received - .saturating_add(received_zat - ironwood_received); + agg.orchard_received = agg.orchard_received.saturating_add(orchard_received); if !memos.is_empty() { - agg.memos.extend(memos); + for memo in memos { + if !agg.memos.contains(&memo) { + agg.memos.push(memo); + } + } } #[cfg(feature = "private-enhance")] self.reconcile_private_enhancement(&txid); } + self.recompute_pool_fields(); // Unlock-on-store: the spend records now guard these notes. self.prune_locks(); Ok(()) diff --git a/crates/z-transparent-pir-v2/Cargo.toml b/crates/z-transparent-pir-v2/Cargo.toml new file mode 100644 index 0000000..6d27739 --- /dev/null +++ b/crates/z-transparent-pir-v2/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "z-transparent-pir-v2" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license = "Apache-2.0 AND MIT" +[dependencies] +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +hex = "0.4" +base64 = "0.22" +sha2 = "0.10" +bincode = "1.3" +futures = "0.3" +transparent-wallet = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-events = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-filter = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-shard = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +transparent-native = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom02 = { package = "getrandom", version = "0.2", features = ["js"] } +getrandom03 = { package = "getrandom", version = "0.3", features = ["wasm_js"] } +[dev-dependencies] +transparent-shard-server = { git = "https://github.com/valargroup/wallet-pir", rev = "c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2" } +tempfile = "3" +tokio = { version = "1", features = ["rt-multi-thread","sync","time"] } +axum = "0.7" +tower = { version = "0.5", features = ["util"] } diff --git a/crates/z-transparent-pir-v2/DESIGN.md b/crates/z-transparent-pir-v2/DESIGN.md new file mode 100644 index 0000000..b67aa6e --- /dev/null +++ b/crates/z-transparent-pir-v2/DESIGN.md @@ -0,0 +1,177 @@ +# Transparent v12 acquisition contract + +The crate preserves the c9262e8 Rust walker/codec/crypto and converts only its +network suspension points to a genuinely asynchronous walker. A Pass owns that +future. `advance()` polls it and returns one owned Step; `accept(request_id, +reply)` supplies a single bounded response; no synchronous replay/cache loop. + +Public serde wire contract (snake_case `kind`, deny unknown fields): +Operation = init | map | filter {shard_id:u64} | +manifest {shard_id,revision:hex64} | +setup {shard_id,revision,table,segment:u32} | +query {shard_id,revision,table,body:base64}. +Table = {kind:"directory"} | {kind:"pages",child:u32}, child <32. +Reply = data {body:base64} | stale {map_digest_hint?:hex64} | +overloaded {retry_after?:u64} | misrouted | unavailable | refused. +Only these six public routes are admitted. Query never selects a segment. + +`Pass::new(scope: Scope, checkpoint: Option, limits: Limits)`; +`advance()->Result`; +`accept(request_id:u64, reply:Reply)->Result<()>`; +`acknowledge(checkpoint_id:&str)->Result<()>`; +`finish_budget()->Result<()>` (active resource deadline only); +`cancel()` (user/session cancellation, never persistence). +Step = Exchange {request_id,operation} | NeedChain {height,hash} | +Checkpoint {checkpoint:Checkpoint} | Stopped {status:Status}. + +Scope serde fields: source_id:String (stable digest), generation:u64, +scope_id:String (wallet inventory digest), anchor:{height:u64,hash:String}, +scripts:[{script:String(hex),required_from:u64}], +accepted_headers:[{height:u64,hash:String}]. The wallet supplies its independently +accepted chain, not publisher metadata. Missing historical points yield NeedChain; +accept_chain(height,hash) only accepts independently validated wallet evidence. + +Checkpoint serde fields: source_id, generation, scope_id, +checkpoint_id:String, previous_checkpoint_id:Option, +anchor:{height,hash}, covered_through:u64, complete:bool, +pending_pages:u64, unresolved_spends:u64, +records:[{script:String(hex),event:String(hex self-contained TransparentEvent bytes), +shard_id:u64,revision:String}], +coverage:[{script:String(hex),required_from:u64,through:u64,revision:String}], +rewind_to:Option<{height,hash}>, +continuation:String(base64 versioned internal logical journal). +Records are a full source-evidence snapshot, deduplicated by upstream semantics; +the engine projects idempotently in its canonical wallet transaction. Continuation +contains durable source-evidence operations/cursors, not query secrets. No public +balance or spend authorization is exposed. Engine validates source/scope/generation +and accepted anchor before projecting and persisting continuation atomically. +Only after durable acknowledgment may a Complete status be published. Cancellation +or source/generation change retires the future and all private query contexts. + +The private source-evidence journal replays the exact upstream WalletStore +contract. It is not a second wallet balance authority: the canonical WalletDb or +WebWallet is the sole product amount/history owner. Its internal ledger is used +only by inherited completeness checks and never exported as product balance. + +Private query contexts remain opaque Rust memory between Exchange and accept. +Reopen discards contexts and regenerates fresh ciphertext from the acknowledged +logical continuation. Retries and malformed-content dummy obligations remain the +upstream walker's. Busy yields bounded delayed progress, never thread sleep. + + +The private continuation is version 4. Versions 2 and 3 are accepted only after their original checkpoint digest is verified, then migrated without changing the source journal or cover-query obligations. The version-4 envelope produces a new checkpoint ID; it does not reinterpret the old digest. A malformed page run records its remaining +cover-query count on the existing pending item, bound to that script, revision, +and accepted target. The failure count and this obligation commit together before +another exchange is yielded. Reopening pays that debt before another real query +for the entry; a query budget can pay only a returned prefix. A refused round +retains its obligation, matching an honest round whose local replies were lost. +Normal anchor advancement preserves debt under the same revision. Public revision +retirement/reorg handling still retires the corresponding old pending work. +No query bytes, random coins, NativeSecret, or in-flight response context is saved. +Incomplete failure checkpoints never publish completion. Unpublished version-1 +transparent continuations are rejected; existing wallet/payment data is unchanged. + +Coverage walks from the script's inclusive required height using an optional end, +so a range covering genesis alone produces a revision-bearing 0..0 coverage row. +A completing checkpoint must carry every enrolled script through its accepted anchor. + + +Historical chain validation has an independent core resource limit: +`Limits.max_chain_reads` defaults to 32 (also when absent from older JSON). +Each new independent historical read, including prior-anchor revalidation, counts +against it. Reaching the limit produces an acknowledged incomplete checkpoint +with reason `chain_budget`, then stops that pass. The host's separate 64-read +limit remains a defense guard. Header work alone never changes source coverage, +completion, rewind markers, receive/spend facts, or query-cover debt. The first +header-only checkpoint reports no coverage when no evidence exists. + +The continuation persists one map of independently observed public height/hash +pairs, including points that disagree with publisher metadata. ChainView derives +acceptance or rejection by comparison; a stored observation is never a claim +that the publisher is correct. Version-three matched-only checkpoints remain a +valid subset of this evidence and retain their original digest validation. +Source, network genesis and accepted target anchor bind every observation. On +an advancing target, the old anchor is independently revalidated on the current +selected chain before any older observation is inherited. A mismatch discards +that cache and uses fresh observations through the unchanged rollback rules. +Same-height anchor changes also invalidate inheritance; lower targets are refused. +While prior-anchor lineage is undecided, Init/Map failures and resource deadlines +stop without emitting a replacement checkpoint. The canonical owner keeps the +exact prior continuation, facts, coverage and cover debt. Receiving an old-anchor +reply alone does not authorize replacement: the Rust future must process the +independent acceptance/rejection and atomically merge or discard its old cache +before any new-target checkpoint is possible. An unpolled reply or dropped future +therefore cannot erase durable progress. Cancellation still wins without a drain. + +The supported metadata domain is explicit: at most 1024 current shards (2048 +boundary heights), 4096 distinct retained coverage endpoints, and genesis, +current anchor and prior anchor. The resulting observation-map bound is 6147; +network/read/time budgets are unchanged. Cache inheritance is pruned to the +exact needed set before merging. Scope-supplied extra independently accepted +points remain ephemeral until needed, so unrelated supplied points cannot +exhaust the durable map. Old coverage compacts by same-script containment and +exact range identity; cross-script or partial-target endpoints have no 1024 +invariant. Exceeding 4096 retained endpoints stops with +`history_metadata_limit`, without IO, checkpoint replacement, evidence truncation +or wallet-state changes. This is an unsupported historical metadata size for +this client; it requires a future resource-policy/support upgrade. A rescan is +not promised to resolve it, and no fallback is permitted. Oversized serialized +observation maps are rejected before use. No PIR query bytes or randomness are +cached. Acknowledged chain or elapsed-budget progress survives cold restart, +including mismatches needed to reach a verified rollback. + +The local retained-event policy is 65,536 unique receive/spend records. A duplicate +at the bound is accepted; retiring evidence frees capacity. Growth mutations are +validated on a candidate journal before publication. The normal serialized store +bound is 63 MiB. A refused mutation retains every previous record, coverage row, +pending item, cache and anchor, and emits an incomplete checkpoint carrying a +persistent `history_metadata_limit` marker. Cold resume of that marker performs +no network IO. This is a client policy limit, not a publisher/transport failure; +rescan retains the continuation and cannot remove the limit. + +Malformed-content cover obligations arise after a query reply is observed. The +exact failed-walk update may therefore use reserved space: it changes only an +existing pending item's dummy debt and its failure counter, preserving all other +pending fields and all facts/coverage/caches. At most 4096 existing pending rows +with scripts of at most 40 bytes add 266,240 bytes in the actual serializer. +A 512 KiB terminal-store reserve preserves those new obligations, sets the same +terminal marker, and stops before another Exchange can escape. Another 512 KiB +bounds the actual serialized source/anchor/header envelope (at most 6147 points), +so the complete continuation remains below 64 MiB. The publication anchor stamped +by `commit_bounded` has no stored effect in an event-free, coverage-free failure +commit and does not disqualify that reserve. If a checked prefix hits a local +cap after its next head was already found malformed, the walker retains that +known content failure and its child/count, then attaches the obligation to the +last durable pending cursor. Other storage errors keep their original class. Encoder-bound tests cover +both reserves. Pure reads and operations that only shrink state avoid candidate +serialization; growth checks are proportional to the retained journal. + +Older valid continuations already beyond the new store/event policy stop +without IO, replacement checkpoints, truncation or altered wallet intents. Their +full original durable checkpoint stays with the canonical owner. Invalid input +encodings, contradictory publisher data and transient service failures retain +their existing error behavior and do not set the policy marker. + +A host resource deadline may call `finish_budget` while the same wallet session +is still active, then drain only Checkpoint/ack/Stopped. This abandons live IO and +query contexts while preserving previously validated logical progress. A held +checkpoint remains held until its ordinary durable acknowledgment. User lock, +user cancellation or an obsolete generation must use `cancel`, which cannot be +converted into a budget finish. An elapsed deadline may therefore commit a +shorter accepted header prefix without raising either read limit. + +Budget checkpoints retain the full source projection and serialize it once per +bounded pass; large wallet checkpoint cost remains proportional to that journal. +They do not prove the publisher indexed every block. The real-service fixture +first recovers receive/spend facts, then validates more than 130 boundary heights +across cold bounded passes and advancing tips, preserving those facts during +header-only progress. + +An incomplete result with an unchanged durable continuation and identical +source/scope/generation/anchor returns Stopped without rewriting the prior +checkpoint. A changed binding, accepted header, journal or cover obligation still +checkpoints. Complete retains its normal checkpoint/ack path. If the complete +logical payload has the same ID, the validated prior metadata (including receipt +link and rewind) is reused exactly; conflicting metadata is refused. A repeated +upstream walk may increment its source-journal commit counter and produce a new +ID while preserving the same canonical facts. diff --git a/crates/z-transparent-pir-v2/LICENSE-APACHE b/crates/z-transparent-pir-v2/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT b/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT new file mode 100644 index 0000000..1717302 --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-FIXTURE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017-2021 The Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crates/z-transparent-pir-v2/LICENSE-MIT b/crates/z-transparent-pir-v2/LICENSE-MIT new file mode 100644 index 0000000..673df49 --- /dev/null +++ b/crates/z-transparent-pir-v2/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025-2026 Roman Akhtariev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/z-transparent-pir-v2/UPSTREAM.md b/crates/z-transparent-pir-v2/UPSTREAM.md new file mode 100644 index 0000000..1e37b51 --- /dev/null +++ b/crates/z-transparent-pir-v2/UPSTREAM.md @@ -0,0 +1,104 @@ +# Transparent acquisition source provenance + +The production acquisition code uses wallet-pir commit +`c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2`, the SDK's existing compatible v12 pin. +Native ReinspiRING, record codecs, filters, tags and manifests remain dependencies +at that commit. This crate implements no cryptographic arithmetic. + +Six private modules adapt `transparent/crates/transparent-wallet/src`: +`adapters.rs`, `client.rs`, `store.rs`, `memory_store.rs`, `sync.rs` and +`sync_ahead.rs`. Their original unit-test sections are omitted. The patch adds +async functions and await points, replaces synchronous transport traits with +local async traits, and changes the refusal closure to `AsyncFnMut`. It retains +the upstream directory/page walk, content-failure dummy requests, failure caps, +revision/re-cut rules and coverage checks. The initial driver permits one +outstanding exchange. The unused one-shot balance facade, default chain helper, +backoff sleep and unused store helpers are removed. Product balances remain in +the owning engine. + +An overloaded read returns incomplete immediately. The host schedules the next +bounded pass. The synchronous upstream facade instead sleeps and retries inside +its pass. No sleep, synchronous XHR, Atomics wait or synchronous replay loop is +used here. Schema/segment/size ceilings are additional host-policy refusals. + +The source-evidence store adds bounded versioned serialization. Events use the +upstream self-contained event codec. Receive/spend indexes rebuild from checked +events after restore. Query secrets and live futures are never serialized. +Rollback records an explicit independently accepted ancestor. Acknowledged +checkpoints contain a continuation and facts for the canonical wallet owner. + +`upstream-sources.json` pins the original full-file SHA256 values. +`upstream.patch` describes changes from those six files after test removal. +To reconstruct and verify the adaptation against an existing local checkout: + +```sh +python3 crates/z-transparent-pir-v2/tools/verify-upstream.py \ + /path/to/wallet-pir-at-c9262e8 +``` + +The command checks the full source hashes, strips only the original test +sections, applies the committed patch in a temporary directory and compares each +result with this crate. No checkout or dependency source is modified. Replace +this adaptation when upstream supplies an equivalent yielding or async walker +that passes the same native/WASM, differential and canonical-wallet fixtures. + +The fixture publisher and in-process server adapter in `tests/fixture.rs` adapt +`zakura/pir-transparent/src/testing/service.rs` at wallet-libraries commit +`3acb18a2f3eaff8f4739d1b26edc73095b80ff26`. The local changes allow a test to name +its independently known network/genesis, support start height zero and give cold +correctness fixtures a 180-second query deadline. This timeout is not a product +capacity or latency claim. The fixture opens no socket. Its MIT notice is in +`LICENSE-FIXTURE-MIT`. + +New Coffer SDK code is Apache-2.0. Adapted wallet-pir modules retain Roman +Akhtariev's MIT notice in each file and in `LICENSE-MIT`. The package's combined +SPDX expression is `Apache-2.0 AND MIT`. `LICENSE-APACHE` contains Apache-2.0. + +Core tests use a real native PIR server against nonzero receive/spend data, +upstream MemoryStore as a separate protocol oracle, and literal fixture events. +The engine integration adds an independent canonical full-block oracle. Fixture +agreement cannot prove that a production publisher has indexed every block. + + +The yielding adaptation additionally makes malformed-content dummy debt durable +on PendingPages. This is a Coffer suspension-boundary extension: the upstream +one-shot Failures.owed vector is not sufficient after an acknowledged checkpoint +or an out-of-call overload. Only logical child/count debt is saved, never crypto +contexts. Tests exercise the real server with malformed but digest-bound page +heads, partial budgets, cancellation, overload and cold anchor advancement. +The wrapper also uses inclusive optional coverage accumulation for genesis-only +ranges; saturating_sub(1) at zero is zero, not an unsigned wraparound. + +The Coffer suspension wrapper persists independently observed canonical-chain +points, including publisher mismatches, in one anchor-bound map. This is public +chain evidence, not a publisher-verification assertion. It prevents bounded cold +passes from repeating rejected old coverage endpoints before upstream rollback. +The explicit supported domain is 1024 current shards and 4096 distinct old +coverage endpoints plus three anchor/genesis points (6147 observations); excess +old metadata fails incomplete without altering the evidence journal. The actual +136-old-shard to one-wide-shard fork fixture records elapsed and chain-budget +cold passes and requires the unchanged upstream walker to reach verified rewind. + +Before prior-anchor lineage is decided, the wrapper emits no replacement +checkpoint on Init/Map failure or resource interruption. The canonical owner +retains the exact prior continuation. Cache inheritance/invalidation becomes +checkpointable only after Rust processes the independent anchor decision. This +is a suspension-boundary rule; upstream validation and rollback are unchanged. + +The suspension profile also bounds retained records and serialized store growth. +The adapted MemoryStore validates the unique-event cap before mutation; its +runtime-only cap does not change the stored MemoryStore encoding. A typed local +policy refusal becomes a durable version-4 `history_metadata_limit` checkpoint. +The exact existing-pending malformed-content update may use reserved space to +persist newly owed dummy traffic before stopping. It cannot alter cursors, +fragments, facts, coverage or caches through that reserve. Versions 2 and 3 keep +their original digest validation before migration. The full event, byte and +serializer-reserve bounds and tests are described in `DESIGN.md`. + +The reserved failure path accepts the inert publication-anchor stamp added by +`commit_bounded`; coverage, records and cache changes remain forbidden. When a +checked run cannot grow the store after a bad next head was already validated, +only the typed local cap preserves that known content failure and owed query +count. The failure update attaches to the unchanged durable pending cursor. +Real Walker regressions cover stamped anchors and both event/byte refusals with +publisher-built page fragments, followed by terminal checkpoint and cold resume. diff --git a/crates/z-transparent-pir-v2/src/adapters.rs b/crates/z-transparent-pir-v2/src/adapters.rs new file mode 100644 index 0000000..8e6a81d --- /dev/null +++ b/crates/z-transparent-pir-v2/src/adapters.rs @@ -0,0 +1,54 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! What the embedding wallet supplies besides storage. +//! +//! The library does not know the wallet's derivation rules, gap limit or +//! imported keys, and it does not decide which chain is the right one. Both +//! are handed in through these traits, so the ledger this library builds +//! follows the wallet's own policies rather than inventing parallel ones. + +use crate::store::{Anchor, ScriptEntry}; + +/// Whether a block the wallet's coverage rests on is still on its chain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Acceptance { + Accepted, + Rejected, + /// The wallet cannot say: it has no view of this height yet. + Unknown, +} + +/// The wallet's view of the chain. Independent of the shard service by +/// construction, which is what lets a reorg be detected rather than accepted. +pub trait ChainView { + fn is_accepted(&self, height: u64, hash_display_hex: &str) -> Acceptance; + fn tip(&self) -> Option; + /// An independently accepted hash, needed for rollback inside a shard. + fn hash_at(&self, height: u64) -> Option { + self.tip().filter(|a| a.height == height).map(|a| a.hash) + } +} + +/// The wallet's script set and its rule for extending it. +pub trait ScriptProvider { + /// Every script the wallet is currently responsible for, with the height + /// its coverage must begin at. + fn scripts(&mut self) -> Vec; + /// Called with the scripts that showed activity in a sync. Returns any + /// scripts the wallet's own rules now add — a gap-limit advance — each + /// with its own required height. Those receive historical discovery + /// over their whole required range before the sync completes. + fn on_activity(&mut self, active: &[Vec]) -> Vec; +} + +/// A fixed script set that never advances. +pub struct StaticScripts(pub Vec); + +impl ScriptProvider for StaticScripts { + fn scripts(&mut self) -> Vec { + self.0.clone() + } + fn on_activity(&mut self, _: &[Vec]) -> Vec { + Vec::new() + } +} diff --git a/crates/z-transparent-pir-v2/src/client.rs b/crates/z-transparent-pir-v2/src/client.rs new file mode 100644 index 0000000..1b80dae --- /dev/null +++ b/crates/z-transparent-pir-v2/src/client.rs @@ -0,0 +1,456 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Private retrieval of directory rows and pages from a shard set. +//! +//! One `TableClient` per geometry per table, reused across every shard naming +//! that geometry. That reuse is the point of naming geometries from a closed +//! registry rather than letting each shard choose: the expensive part of a +//! session — the parameter re-derivation and the public query setup — is done +//! once per geometry rather than once per shard. Both are the native +//! ReinspiRING two-mask profile of `transparent-native`. A sync spanning an archive +//! tier and a recent tier pays it twice, not once per shard in either. +//! +//! What *is* per segment is the published pair of masks, because it is derived +//! from that segment's own database. So a shard *revision* is "opened" by fetching the +//! setup of each of its segments, and those bytes are charged once per segment +//! per sync. A shard normally has one segment; one whose content did not fit +//! its geometry has more, and every one of them answers each query, so the +//! segment holding a script is never named in a request. +//! +//! Setup is cached against the **manifest digest**, not the shard id. A growing +//! tail is republished as a new revision with different bytes, so setup cached +//! under the id would be reused against a database it was not derived from — +//! which does not error, it decodes to plausible nonsense. +//! +//! Everything the server sends is re-derived rather than trusted. A client that +//! adopted the server's parameters would decode against whatever geometry the +//! server chose, including one that leaks the selection. +//! +//! The directory and pages use the 8-bit profile, whose one scheme is a +//! 28-bit dithered query. A 16-bit table publishes a 49-bit scheme and may +//! advertise a 44-bit dithered one beside it; the query uses the 44 bits only +//! when that scheme reproduces exactly, and otherwise the 49 bits, which are +//! still checked whole. + +use crate::transport::{BoxError, ByteCharges, Overloaded, ShardTransport, StaleRevision}; +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; +use transparent_native::{NativeScheme, NativeSecret, TableProfile}; + +#[derive(Debug, thiserror::Error)] +pub enum ClientError { + #[error("session: {0}")] + Session(String), + #[error("response: {0}")] + Response(String), + #[error("transport: {0}")] + Transport(String), + #[error("pir: {0}")] + Pir(String), + /// The service no longer serves the revision this query named. + /// + /// Kept as a type rather than folded into `Transport`: it is recoverable by + /// refreshing the map and re-deriving the shard, and a string cannot be + /// matched on. Every query goes through here, so collapsing it here would + /// lose the refusal for all of them. + #[error(transparent)] + Stale(#[from] StaleRevision), + /// The service had no cache capacity for this query. Retryable as it + /// stands, without refreshing anything. + #[error(transparent)] + Overloaded(#[from] Overloaded), + /// A reply was made under other published parameters than the setup the + /// wallet holds for that segment. The held setup is stale: the caller + /// drops it, so the next attempt fetches the published one again. + #[error("revision {revision} table {} segment {segment}: the reply was made under other parameters than the setup held", .table.name())] + SetupMismatch { + revision: String, + table: Table, + segment: u32, + }, +} + +pub use transparent_wallet::client::Table; + +/// A prepared query and the secret needed to decode its response. +pub struct PreparedQuery { + pub body: Vec, + secret: NativeSecret, +} + +/// One segment's published setup for one table: its two rounded masks, as +/// downloaded, and the epoch they imply. +struct SegmentSetup { + public: Vec, + epoch: [u8; 8], +} + +/// A client for one table kind of one geometry, across every shard. +/// +/// One pages client serves every `pages-` table: the scheme is per kind, +/// while each table's published setup, binding and stash are per table name. +pub struct TableClient { + schema: String, + /// The kind this client was derived for; see [`Table::kind`]. + kind: &'static str, + profile: TableProfile, + /// Whether queries go at 44 dithered bits, which the service advertised + /// and this client reproduced, rather than 49. + dithered: bool, + /// Per segment, because the published masks come from each segment's own + /// database. Keyed by manifest digest, table name and segment index, so a + /// superseded revision's setup is never reused for the revision that + /// replaced it, and one page shard's never for another's. + segments: HashMap<(String, String, u32), SegmentSetup>, + /// Queries a batch already sent, per revision and table, in the order the + /// walk will ask for their rows. See [`TableClient::stash`]. + stashed: HashMap<(String, String), VecDeque>, +} + +/// A query sent ahead of the walk, with its outcome. +struct Stashed { + row: usize, + query: PreparedQuery, + reply: Result, BoxError>, +} + +impl TableClient { + /// Builds a client for one table kind of `geometry`, re-deriving the + /// native parameters — bit widths, query masks and packing setup — rather + /// than adopting what the service published. + /// + /// `table` names the kind; any `Table::Pages(_)` builds the client every + /// page table of the geometry shares. `served` is the 49-bit scheme, + /// which must reproduce. `served_dithered` is the 44-bit dithered scheme, + /// if the service advertised one: queries use it only when it reproduces + /// too, and otherwise fall back to 49 bits. + + pub fn new_with_schema( + schema: &str, + table: Table, + geometry: &str, + rows: u64, + row_bytes: u32, + served: &NativeScheme, + served_dithered: Option<&NativeScheme>, + ) -> Result { + if !transparent_shard::manifest::supported_schema(schema) { + return Err(ClientError::Session("unsupported schema".into())); + } + let kind = table.kind(); + let profile = + TableProfile::new(schema, geometry, kind, rows, row_bytes).map_err(ClientError::Pir)?; + if served != &profile.scheme { + return Err(ClientError::Session(format!( + "{kind} scheme does not match the pinned geometry" + ))); + } + // Compared whole, like the 49-bit scheme. One that does not reproduce + // is not adopted; the query falls back to the 49 bits checked above. + let dithered = profile + .dithered_scheme + .as_ref() + .is_some_and(|scheme| served_dithered == Some(scheme)); + Ok(Self { + schema: schema.to_string(), + kind, + profile, + dithered, + segments: HashMap::new(), + stashed: HashMap::new(), + }) + } + + pub fn rows(&self) -> usize { + self.profile.rows + } + + /// Bits each selection coefficient is sent at: the scheme's own (28 for + /// the 8-bit tables, 49 for a 16-bit one), or 44 when a 16-bit table's + /// service advertised the dithered scheme and it reproduced. + pub fn query_bits(&self) -> usize { + match &self.profile.dithered_scheme { + Some(scheme) if self.dithered => scheme.query_bits, + _ => self.profile.scheme.query_bits, + } + } + + fn check_kind(&self, table: Table) -> Result<(), ClientError> { + if table.kind() != self.kind { + return Err(ClientError::Session(format!( + "a {} client cannot address table {}", + self.kind, + table.name() + ))); + } + Ok(()) + } + + /// Records one segment's published setup after checking it against its + /// digest. + pub fn open_segment( + &mut self, + revision: &str, + table: Table, + segment: u32, + public_params_base64: &str, + public_params_sha256: &str, + ) -> Result<(), ClientError> { + self.check_kind(table)?; + let public_params = BASE64_STANDARD + .decode(public_params_base64.as_bytes()) + .map_err(|error| ClientError::Session(error.to_string()))?; + let digest = Sha256::digest(&public_params); + if hex::encode(digest) != public_params_sha256 { + return Err(ClientError::Session(format!( + "revision {revision} published parameter digest mismatch" + ))); + } + if public_params.len() != self.profile.scheme.public_bytes { + return Err(ClientError::Session(format!( + "revision {revision} published parameter length mismatch" + ))); + } + let mut epoch = [0u8; 8]; + epoch.copy_from_slice(&digest[..8]); + self.segments.insert( + (revision.to_string(), table.name(), segment), + SegmentSetup { + public: public_params, + epoch, + }, + ); + Ok(()) + } + + /// Drops one segment's setup, so it is opened again before the next query. + pub(crate) fn forget(&mut self, revision: &str, table: Table, segment: u32) { + self.segments + .remove(&(revision.to_string(), table.name(), segment)); + } + + pub fn is_open(&self, revision: &str, table: Table, segment: u32) -> bool { + self.segments + .contains_key(&(revision.to_string(), table.name(), segment)) + } + + /// Prepares one query against a named revision and table. + /// + /// The body opens with the revision-and-table binding, which the service + /// re-derives and checks and the response repeats. It is a fixed-width + /// public value — the revision is already in the request path and in the + /// published map — so it discloses nothing while making a query answered by + /// the wrong runtime fail rather than decode. A fresh secret is sampled for + /// every query and never leaves the returned value. + pub fn prepare( + &self, + revision: &str, + table: Table, + row: usize, + ) -> Result { + self.prepare_prefix(revision, table, self.profile.rows, row) + } + + /// Prepares one query that selects among only the first `query_rows`, + /// which the revision's verified manifest fixes for every query to the + /// table (see [`PageChild::query_rows`]). Never derived from `row`: the + /// upload's length is all the service learns from it. + /// + /// [`PageChild::query_rows`]: transparent_shard::manifest::PageChild::query_rows + pub fn prepare_prefix( + &self, + revision: &str, + table: Table, + query_rows: usize, + row: usize, + ) -> Result { + self.check_kind(table)?; + if row >= query_rows || query_rows > self.profile.rows { + return Err(ClientError::Session("row outside table".into())); + } + let (secret, upload) = if self.dithered { + self.profile.prepare_dithered_prefix(query_rows, row) + } else { + self.profile.prepare_prefix(query_rows, row) + } + .map_err(ClientError::Pir)?; + let mut body = transparent_shard::manifest::query_binding_for_schema( + &self.schema, + revision, + &table.name(), + ) + .to_vec(); + body.extend(upload); + Ok(PreparedQuery { body, secret }) + } + + /// Decodes one answer: one row per segment, in segment order. + /// + /// Every segment answered the same query, so the response carries as many + /// bodies as the table has segments and the caller decides which row it + /// was looking for — by the exact script tag, never by position. + pub fn decode( + &self, + revision: &str, + table: Table, + segments: u32, + query: PreparedQuery, + response: &[u8], + ) -> Result>, ClientError> { + self.check_kind(table)?; + let name = table.name(); + let binding = + transparent_shard::manifest::query_binding_for_schema(&self.schema, revision, &name); + let each = 16 + self.profile.scheme.response_bytes; + // A fixed length per segment, and a fixed segment count from the + // verified manifest: a response of any other size is not this table's. + if response.len() != each * segments as usize { + return Err(ClientError::Response("response length mismatch".into())); + } + + let mut rows = Vec::with_capacity(segments as usize); + for segment in 0..segments { + let part = &response[segment as usize * each..(segment as usize + 1) * each]; + let setup = self + .segments + .get(&(revision.to_string(), name.clone(), segment)) + .ok_or_else(|| { + ClientError::Session(format!( + "revision {revision} table {name} segment {segment} is not open" + )) + })?; + // The prefix binds the response to the revision and table the query + // named, so an answer produced from another revision's bytes fails + // here rather than decoding to plausible rows for a range the + // wallet did not ask about. The epoch does the same for the + // segment, since each segment publishes its own masks. + if part.get(..8) != Some(binding.as_slice()) { + return Err(ClientError::Response("revision or table mismatch".into())); + } + if part.get(8..16) != Some(setup.epoch.as_slice()) { + return Err(ClientError::SetupMismatch { + revision: revision.to_string(), + table, + segment, + }); + } + let row = self + .profile + .decode(&query.secret, &setup.public, &part[16..]) + .map_err(ClientError::Response)?; + if row.len() != self.profile.row_bytes { + return Err(ClientError::Response("decoded row is too short".into())); + } + rows.push(row); + } + Ok(rows) + } + + /// Keeps a query that was sent ahead of the walk, with its outcome, for + /// the walk's next [`fetch_row`](Self::fetch_row) of that revision and + /// table. + /// + /// Stashed queries are handed out once, in order, and only for the row + /// they were prepared for: the first request that does not match drops + /// the stash and goes to the network. A stashed refusal or failure is + /// returned as if the query had just been sent, and drops the rest of the + /// stash, so whatever the walk does next — a retry, a refresh — is a + /// fresh request. + pub(crate) fn stash( + &mut self, + revision: &str, + table: Table, + row: usize, + query: PreparedQuery, + reply: Result, BoxError>, + ) { + self.stashed + .entry((revision.to_string(), table.name())) + .or_default() + .push_back(Stashed { row, query, reply }); + } + + /// Drops every stashed query. + pub(crate) fn clear_stash(&mut self) { + self.stashed.clear(); + } + + /// Fetches one row from every segment of a table, charging what it cost. + /// + /// The query selects among the first `query_rows`; see + /// [`prepare_prefix`](Self::prepare_prefix). Uses a query a batch already + /// sent for this row when one is stashed; the charge is the same either way. + #[allow(clippy::too_many_arguments)] + pub async fn fetch_row( + &mut self, + transport: &mut impl ShardTransport, + shard_id: u64, + revision: &str, + table: Table, + segments: u32, + query_rows: usize, + row: usize, + charges: &mut ByteCharges, + ) -> Result>, ClientError> { + let key = (revision.to_string(), table.name()); + if let Some(queue) = self.stashed.get_mut(&key) { + let next = queue.pop_front(); + if queue.is_empty() { + self.stashed.remove(&key); + } + match next { + Some(stashed) + if stashed.row == row + && stashed.query.body.len() + == 8 + self + .profile + .native + .request_len_bits(query_rows, self.query_bits()) => + { + let response = match stashed.reply { + Ok(response) => response, + Err(error) => { + self.stashed.remove(&key); + return Err(classify_transport(error)); + } + }; + charges.add_query( + table, + stashed.query.body.len() as u64, + response.len() as u64, + ); + return self.decode(revision, table, segments, stashed.query, &response); + } + // Out of step with the walk: nothing further is trusted to + // line up, so the rest of this table goes to the network. + _ => { + self.stashed.remove(&key); + } + } + } + let query = self.prepare_prefix(revision, table, query_rows, row)?; + let uploaded = query.body.len() as u64; + let response = transport + .query(shard_id, revision, table, &query.body) + .await + .map_err(classify_transport)?; + charges.add_query(table, uploaded, response.len() as u64); + self.decode(revision, table, segments, query, &response) + } +} + +/// Sorts a boxed transport error into the refusals a caller can act on. +/// +/// The refusals are recovered before the fallback, because both are boxed into +/// the same `BoxError` as an ordinary failure and stringifying first would +/// throw away the only thing that distinguishes them. +pub(crate) fn classify_transport(error: crate::transport::BoxError) -> ClientError { + if let Some(stale) = StaleRevision::found_in(&error) { + return ClientError::Stale(stale.clone()); + } + if let Some(overloaded) = Overloaded::found_in(&error) { + return ClientError::Overloaded(overloaded.clone()); + } + ClientError::Transport(crate::transport::describe_error(error.as_ref())) +} diff --git a/crates/z-transparent-pir-v2/src/lib.rs b/crates/z-transparent-pir-v2/src/lib.rs new file mode 100644 index 0000000..93ba970 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/lib.rs @@ -0,0 +1,18 @@ +//! Rust-owned, yielding transparent v12 acquisition. No wallet balance authority. +#![allow(async_fn_in_trait)] +mod types; +pub use types::*; +mod adapters; +mod client; +mod memory_store; +mod pass; +mod serialization; +mod store; +mod sync; +mod transport; +pub use pass::{AcceptedRewind, Error, Pass}; +mod ledger { + pub use transparent_wallet::ledger::*; +} + +mod shared_store; diff --git a/crates/z-transparent-pir-v2/src/memory_store.rs b/crates/z-transparent-pir-v2/src/memory_store.rs new file mode 100644 index 0000000..b501689 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/memory_store.rs @@ -0,0 +1,523 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! The store, in memory: the reference semantics every other implementation +//! has to match, and what the one-shot `sync` uses. + +use crate::ledger::LedgerError; +use crate::store::{ + merge_coverage, receive_key, spend_key, Anchor, CachedRun, CoverageKind, CoverageRange, + PendingPages, ScriptEntry, SetIdentity, SetupBlob, SetupKey, ShardCommit, StoreError, + StoredEvent, WalletStore, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use transparent_events::{TransparentEvent, Txid}; + +/// Default bound on pending page retrievals a commit may leave. +pub const DEFAULT_PENDING_LIMIT: usize = 4_096; + +type ReceiveKey = (Txid, u32); +type SpendKey = (Txid, u32, Txid, u32); + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct State { + identity: Option, + anchor: Option, + settled_through: Option, + covered_through: Option, + scripts: BTreeMap, ScriptEntry>, + coverage: BTreeMap, Vec>, + #[serde(with = "crate::serialization::receives")] + receives: BTreeMap, + #[serde(with = "crate::serialization::spends")] + spends: BTreeMap, + /// Which spend consumes each outpoint, for the double-spend rule. + #[serde(skip)] + spent_by: BTreeMap, + pending: BTreeMap, + next_pending: u64, + setups: BTreeMap, + /// Run histories by page shard identity and script. + runs: BTreeMap<(String, Vec), CachedRun>, + filters: BTreeMap)>, + /// Content failures by shard id and script. + failures: BTreeMap<(u64, Vec), u32>, + commits: u64, +} + +/// Private source-evidence store with versioned logical continuation. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct MemoryStore { + state: State, + pending_limit: Option, + // Runtime profile policy; never changes the stored MemoryStore encoding. + #[serde(skip)] + event_limit: Option, +} + +impl MemoryStore { + pub fn new() -> Self { + Self::default() + } + pub(crate) fn set_event_limit(&mut self, limit: usize) { + self.event_limit = Some(limit); + } + + pub(crate) fn retain_scope(&mut self, scripts: &std::collections::BTreeSet>) { + self.state.scripts.retain(|s, _| scripts.contains(s)); + self.state.coverage.retain(|s, _| scripts.contains(s)); + self.state + .receives + .retain(|_, e| scripts.contains(&e.script)); + self.state.spends.retain(|_, e| scripts.contains(&e.script)); + self.state + .pending + .retain(|_, p| scripts.contains(&p.script)); + self.state.runs.retain(|(_, s), _| scripts.contains(s)); + self.rebuild_indexes(); + } + + pub(crate) fn rebuild_indexes(&mut self) { + self.state.spent_by.clear(); + for stored in self.state.spends.values() { + if let TransparentEvent::Spend(event) = stored.event { + self.state.spent_by.insert( + (event.spent_txid, event.spent_output_index), + spend_key(&event), + ); + } + } + } + + fn height_of(event: &TransparentEvent) -> u64 { + match event { + TransparentEvent::Receive(receive) => u64::from(receive.height), + TransparentEvent::Spend(spend) => u64::from(spend.height), + } + } +} + +impl WalletStore for MemoryStore { + fn set_identity(&self) -> Result, StoreError> { + Ok(self.state.identity.clone()) + } + + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + match &self.state.identity { + None => { + self.state.identity = Some(identity.clone()); + Ok(()) + } + Some(stored) if stored.continues(identity) => { + // A geometry the store had not seen may appear; record it. + self.state.identity = Some(identity.clone()); + Ok(()) + } + Some(stored) => Err(StoreError::SetMismatch { + stored: stored.digest(), + offered: identity.digest(), + }), + } + } + + fn anchor(&self) -> Result, StoreError> { + Ok(self.state.anchor.clone()) + } + + fn scripts(&self) -> Result, StoreError> { + Ok(self.state.scripts.values().cloned().collect()) + } + + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + let mut added = 0; + for entry in entries { + match self.state.scripts.get_mut(&entry.script) { + Some(existing) => { + // Coverage once required stays required. + existing.required_from = existing.required_from.min(entry.required_from); + } + None => { + self.state + .scripts + .insert(entry.script.clone(), entry.clone()); + added += 1; + } + } + } + Ok(added) + } + + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + Ok(self.state.coverage.get(script).cloned().unwrap_or_default()) + } + + fn provisional(&self) -> Result, StoreError> { + Ok(self + .state + .coverage + .values() + .flatten() + .filter(|range| range.kind == CoverageKind::Provisional) + .cloned() + .collect()) + } + + fn events(&self) -> Result, StoreError> { + let mut events: Vec = self + .state + .receives + .values() + .chain(self.state.spends.values()) + .cloned() + .collect(); + events.sort_by_key(|entry| entry.event.sort_key()); + Ok(events) + } + + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + transparent_events::check_transaction_consistency( + self.state + .receives + .values() + .chain(self.state.spends.values()) + .chain(commit.events.iter()) + .map(|stored| &stored.event), + ) + .map_err(|e| StoreError::Corrupt(e.to_string()))?; + // Validate everything against the current state first; only then + // write, so a refused commit leaves the store exactly as it was. + let mut new_receives: Vec<(ReceiveKey, StoredEvent)> = Vec::new(); + let mut new_spends: Vec<(SpendKey, StoredEvent)> = Vec::new(); + for stored in &commit.events { + match &stored.event { + TransparentEvent::Receive(receive) => { + let key = receive_key(receive); + if let Some(existing) = self.state.receives.get(&key) { + if existing.event != stored.event || existing.script != stored.script { + return Err(LedgerError::ConflictingReceive( + receive.txid.to_display_hex(), + receive.output_index, + ) + .into()); + } + continue; + } + new_receives.push((key, stored.clone())); + } + TransparentEvent::Spend(spend) => { + let key = spend_key(spend); + let outpoint = (spend.spent_txid, spend.spent_output_index); + if let Some(existing) = self.state.spends.get(&key) { + if existing.event != stored.event || existing.script != stored.script { + return Err(LedgerError::DoubleSpend( + spend.spent_txid.to_display_hex(), + spend.spent_output_index, + ) + .into()); + } + continue; + } + if let Some(other) = self.state.spent_by.get(&outpoint) { + if *other != key { + return Err(LedgerError::DoubleSpend( + spend.spent_txid.to_display_hex(), + spend.spent_output_index, + ) + .into()); + } + } + new_spends.push((key, stored.clone())); + } + } + } + let limit = self.pending_limit(); + let pending_after = self.state.pending.len() + + commit + .pending_upsert + .iter() + .filter(|p| p.id.is_none()) + .count() + - commit.pending_complete.len().min(self.state.pending.len()); + if pending_after > limit { + return Err(StoreError::PendingLimit { limit }); + } + for run in &commit.cached_runs { + if self + .state + .runs + .get(&(run.child_digest.clone(), run.script.clone())) + .is_some_and(|held| held != run) + { + return Err(StoreError::Corrupt(format!( + "page shard {} holds two histories for one script", + run.child_digest + ))); + } + } + + if let Some(limit) = self.event_limit { + let receives: std::collections::BTreeSet<_> = + new_receives.iter().map(|(key, _)| *key).collect(); + let spends: std::collections::BTreeSet<_> = + new_spends.iter().map(|(key, _)| *key).collect(); + if self + .state + .receives + .len() + .saturating_add(self.state.spends.len()) + .saturating_add(receives.len()) + .saturating_add(spends.len()) + > limit + { + return Err(StoreError::HistoryMetadataLimit); + } + } + + for (key, event) in new_receives { + self.state.receives.insert(key, event); + } + for (key, event) in new_spends { + let outpoint = (key.2, key.3); + self.state.spent_by.insert(outpoint, key); + self.state.spends.insert(key, event); + } + let kind = if commit.sealed { + CoverageKind::Settled + } else { + CoverageKind::Provisional + }; + for script in &commit.covered_scripts { + let ranges = self.state.coverage.entry(script.clone()).or_default(); + let range = CoverageRange { + source_anchor: commit.source_anchor.clone(), + script: script.clone(), + start_height: commit.start_height, + end_height: commit.end_height, + kind, + shard_id: commit.shard_id, + revision_digest: commit.revision_digest.clone(), + terminal_block_hash: commit.terminal_block_hash.clone(), + }; + // A range strictly inside the one starting nearest at or below it + // is already covered: nothing changes. Otherwise it replaces every + // range of this script it contains, whatever shard they came + // from: a repeat, a longer revision at the same start, and the + // narrower shards a re-cut merged into this one. The SQLite store + // keeps the same rule. + let holder = ranges + .iter() + .filter(|old| old.start_height <= range.start_height) + .max_by_key(|old| (old.start_height, old.end_height)); + if holder.is_some_and(|old| { + old.end_height >= range.end_height + && (old.start_height, old.end_height) != (range.start_height, range.end_height) + }) { + continue; + } + ranges.retain(|old| { + !(old.start_height >= range.start_height && old.end_height <= range.end_height) + }); + if !ranges.contains(&range) { + ranges.push(range); + } + let merged = merge_coverage(std::mem::take(ranges)); + *ranges = merged; + } + for id in &commit.pending_complete { + self.state.pending.remove(id); + } + let finishes_shard = commit.finishes_shard(); + for run in commit.cached_runs { + self.state + .runs + .insert((run.child_digest.clone(), run.script.clone()), run); + } + if finishes_shard { + self.state.runs.retain(|(_, script), run| { + run.shard_id != commit.shard_id || !commit.covered_scripts.contains(script) + }); + } + for script in commit.failed_entries { + *self + .state + .failures + .entry((commit.shard_id, script)) + .or_default() += 1; + } + for mut pending in commit.pending_upsert { + let id = match pending.id { + Some(id) => id, + None => { + self.state.next_pending += 1; + self.state.next_pending + } + }; + pending.id = Some(id); + self.state.pending.insert(id, pending); + } + self.state.commits += 1; + Ok(self.state.commits) + } + + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + self.state.anchor = Some(anchor.clone()); + self.state.settled_through = Some(settled_through); + self.state.covered_through = Some(covered_through); + self.state.commits += 1; + Ok(self.state.commits) + } + + fn rollback_above(&mut self, accepted: &Anchor, _reason: &str) -> Result { + let height = accepted.height; + self.state + .receives + .retain(|_, stored| Self::height_of(&stored.event) <= height); + let mut removed_spends: Vec = Vec::new(); + self.state.spends.retain(|key, stored| { + let keep = Self::height_of(&stored.event) <= height; + if !keep { + removed_spends.push(*key); + } + keep + }); + for key in removed_spends { + self.state.spent_by.remove(&(key.2, key.3)); + } + for ranges in self.state.coverage.values_mut() { + ranges.retain_mut(|range| { + if range.start_height > height { + return false; + } + if range.end_height > height { + // A range straddling the rollback keeps its covered part; + // the identity it carried no longer describes the whole + // range, but it still names the shard the part came from. + range.end_height = height; + range.terminal_block_hash = accepted.hash.clone(); + } + true + }); + } + // Cached runs survive: each is keyed by its page shard's chained + // identity, which commits to that page shard's terminal block and to + // every earlier one, so a walk reaches it only through a manifest on + // the accepted chain. A tail replaced by a later revision keeps the + // runs its sealed page shards share, which is what lets the catch-up + // stop at them. + // All pending retrievals are inexpensive to re-plan and may contain old-branch data. + self.state.pending.clear(); + if let Some(anchor) = &mut self.state.anchor { + if anchor.height > height { + anchor.height = height; + anchor.hash = accepted.hash.clone(); + } + } + if let Some(settled) = &mut self.state.settled_through { + *settled = (*settled).min(height); + } + if let Some(covered) = &mut self.state.covered_through { + *covered = (*covered).min(height); + } + self.state.commits += 1; + Ok(self.state.commits) + } + + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + for ranges in self.state.coverage.values_mut() { + for range in ranges.iter_mut() { + if range.shard_id == shard_id + && range.revision_digest == revision_digest + && range.kind == CoverageKind::Provisional + { + range.kind = CoverageKind::Settled; + } + } + let merged = merge_coverage(std::mem::take(ranges)); + *ranges = merged; + } + Ok(()) + } + + fn pending(&self) -> Result, StoreError> { + Ok(self.state.pending.values().cloned().collect()) + } + + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + Ok(self + .state + .runs + .get(&(child_digest.to_string(), script.to_vec())) + .cloned()) + } + + fn pending_limit(&self) -> usize { + self.pending_limit.unwrap_or(DEFAULT_PENDING_LIMIT) + } + + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + Ok(self.state.setups.get(key).cloned()) + } + + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + self.state.setups.insert(key.clone(), blob.clone()); + Ok(()) + } + + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + self.state.setups.remove(key); + Ok(()) + } + + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + Ok(self + .state + .filters + .get(revision_digest) + .filter(|(hash, _, _)| hash == filter_hash) + .map(|(_, _, bytes)| bytes.clone())) + } + + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + self.state.filters.insert( + revision_digest.to_string(), + (filter_hash.to_string(), sealed, bytes.to_vec()), + ); + Ok(()) + } + + fn last_commit(&self) -> Result { + Ok(self.state.commits) + } + + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + Ok(self + .state + .failures + .get(&(shard_id, script.to_vec())) + .copied() + .unwrap_or(0)) + } +} diff --git a/crates/z-transparent-pir-v2/src/pass.rs b/crates/z-transparent-pir-v2/src/pass.rs new file mode 100644 index 0000000..a23d5b6 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/pass.rs @@ -0,0 +1,1233 @@ +use crate::{ + adapters::{Acceptance, ChainView, StaticScripts}, + client::Table as NativeTable, + memory_store::MemoryStore, + shared_store::{SharedStore, MAX_CONTINUATION_BYTES, MAX_RETAINED_EVENTS, MAX_STORE_BYTES}, + store::{ScriptEntry, ScriptOrigin, WalletStore}, + sync::{self, Completion, GeometryParams, ServiceGeometry, WorkLimits}, + transport::{BoxError, FilterSource, Overloaded, ShardTransport, StaleRevision}, + types::*, +}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use bincode::Options; +use futures::{ + future::{poll_fn, LocalBoxFuture}, + task::noop_waker_ref, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + cell::{Cell, RefCell}, + collections::BTreeMap, + rc::Rc, + task::{Context, Poll}, + time::Duration, +}; +#[derive(Serialize, Deserialize)] +struct LegacyContinuation { + version: u32, + source_id: String, + state: MemoryStore, +} +// Independently observed selected-chain points, including publisher mismatches. +// Comparison with a publication is derived by ChainView, never persisted as trust. +const MAX_MAP_SHARDS: usize = 1024; +const MAX_OLD_ENDPOINTS: usize = 4096; +const MAX_OBSERVED_POINTS: usize = 2 * MAX_MAP_SHARDS + MAX_OLD_ENDPOINTS + 3; +#[derive(Clone, Serialize, Deserialize)] +struct ObservedChain { + anchor: Anchor, + genesis: Option, + #[serde(deserialize_with = "bounded_headers")] + headers: BTreeMap, +} +fn bounded_headers<'de, D: serde::Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + struct Headers; + impl<'de> serde::de::Visitor<'de> for Headers { + type Value = BTreeMap; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a bounded map of independently observed headers") + } + fn visit_map>( + self, + mut map: M, + ) -> std::result::Result { + if map.size_hint().is_some_and(|n| n > MAX_OBSERVED_POINTS) { + return Err(serde::de::Error::custom("header limit")); + } + let mut headers = BTreeMap::new(); + while let Some((height, hash)) = map.next_entry()? { + if headers.len() >= MAX_OBSERVED_POINTS || headers.insert(height, hash).is_some() { + return Err(serde::de::Error::custom("header limit")); + } + } + Ok(headers) + } + } + d.deserialize_map(Headers) +} +#[derive(Serialize, Deserialize)] +struct LegacyContinuationV3 { + version: u32, + source_id: String, + state: MemoryStore, + chain: ObservedChain, +} +#[derive(Serialize, Deserialize)] +struct Continuation { + version: u32, + source_id: String, + state: MemoryStore, + chain: ObservedChain, + policy_limit: bool, +} +// Coverage compaction is per script and revision: old recuts and partial +// targets across scripts have no map-size bound. Refuse an explicit supported +// metadata limit without truncating evidence or asking the walker to guess. +fn retained_endpoints(state: &MemoryStore) -> Result> { + let mut endpoints = std::collections::BTreeSet::new(); + for script in state.scripts().map_err(|_| Error("store"))? { + for range in state.coverage(&script.script).map_err(|_| Error("store"))? { + endpoints.insert(range.end_height); + if endpoints.len() > MAX_OLD_ENDPOINTS { + return Err(Error("history_metadata_limit")); + } + } + } + Ok(endpoints) +} +fn needed_points( + boundaries: impl IntoIterator, + old: &std::collections::BTreeSet, + current: u64, + prior: Option, +) -> std::collections::BTreeSet { + let mut needed = std::collections::BTreeSet::from([0, current]); + needed.extend(prior); + needed.extend(old); + needed.extend(boundaries); + needed +} +fn reason_code(r: sync::IncompleteReason) -> &'static str { + use sync::IncompleteReason::*; + match r { + QueryBudget => "query_budget", + ByteBudget => "byte_budget", + PendingLimit => "pending_limit", + Overloaded { .. } => "overloaded", + ChainUnknown { .. } => "chain_unknown", + DiscoveryUnbounded => "scope_growing", + ContentFailed => "content_failed", + PublicationBehind { .. } => "publication_lag", + UnresolvedSpends => "unresolved_spends", + } +} + +#[derive(Debug, thiserror::Error)] +#[error("transparent recovery: {0}")] +pub struct Error(pub(crate) &'static str); +type Result = std::result::Result; +#[derive(Clone)] +enum Pending { + Exchange { id: u64, operation: Operation }, + Chain { height: u64, hash: String }, +} +#[derive(Default)] +struct Io { + next: u64, + pending: Option, + reply: Option, + chain_reply: Option, + exchanges: u64, + chain_reads: u64, + download: u64, + upload: u64, + stop: Option<&'static str>, +} +#[derive(Clone)] +struct Driver { + io: Rc>, + limits: Limits, + chain: Rc>>, + supplied: Rc>, +} +impl Driver { + fn retain_points(&self, needed: &std::collections::BTreeSet) { + self.chain + .borrow_mut() + .retain(|height, _| needed.contains(height)); + } + async fn send(&self, operation: Operation) -> std::result::Result, BoxError> { + let upload = if let Operation::Query { body, .. } = &operation { + body.len() as u64 + } else { + 0 + }; + { + let mut io = self.io.borrow_mut(); + if io.exchanges >= self.limits.max_exchanges + || io.upload.saturating_add(upload) > self.limits.max_upload_bytes + { + io.stop = Some("budget"); + return Err(Box::new(Error("budget"))); + } + io.exchanges += 1; + io.upload += upload; + io.next += 1; + let id = io.next; + io.pending = Some(Pending::Exchange { + id, + operation: operation.clone(), + }); + } + let reply = poll_fn(|_| { + let mut io = self.io.borrow_mut(); + match io.reply.take() { + Some(r) => { + io.pending = None; + Poll::Ready(r) + } + None => Poll::Pending, + } + }) + .await; + match reply { + Reply::Data { body } => { + let max = match operation { + Operation::Filter { .. } => 8 * 1024 * 1024, + Operation::Setup { .. } => 128 * 1024, + Operation::Query { .. } => 512 * 1024, + _ => 1024 * 1024, + }; + if body.len() > max * 4 / 3 + 8 { + return Err(Box::new(Error("response_limit"))); + } + let bytes = STANDARD + .decode(body) + .map_err(|_| Error("response_encoding"))?; + if bytes.len() > max { + return Err(Box::new(Error("response_limit"))); + } + Ok(bytes) + } + Reply::Stale { map_digest_hint } => { + let (shard_id, revision) = match operation { + Operation::Manifest { shard_id, revision } + | Operation::Setup { + shard_id, revision, .. + } + | Operation::Query { + shard_id, revision, .. + } => (shard_id, revision), + _ => (0, String::new()), + }; + Err(Box::new(StaleRevision { + shard_id, + revision, + map_sha256: map_digest_hint, + })) + } + Reply::Overloaded { retry_after } => Err(Box::new(Overloaded { + retry_after: retry_after.map(|s| Duration::from_secs(s.clamp(1, 60))), + })), + Reply::Misrouted => Err(Box::new(Error("misrouted"))), + Reply::Unavailable => Err(Box::new(Error("unavailable"))), + Reply::Refused => Err(Box::new(Error("refused"))), + } + } + async fn point(&self, height: u64, expected: &str) -> Result<()> { + if self.chain.borrow().contains_key(&height) { + // Existing independently accepted points remain authoritative; the + // walker must see Rejected for obsolete coverage on a reorg. + return Ok(()); + } + if self.chain.borrow().len() >= MAX_OBSERVED_POINTS { + return Err(Error("chain_limit")); + } + if let Some(observed) = self.supplied.get(&height) { + self.chain.borrow_mut().insert(height, observed.clone()); + return Ok(()); + } + { + let mut io = self.io.borrow_mut(); + if io.chain_reads >= self.limits.max_chain_reads { + io.stop = Some("chain_budget"); + return Err(Error("chain_budget")); + } + io.chain_reads += 1; + } + self.io.borrow_mut().pending = Some(Pending::Chain { + height, + hash: expected.to_string(), + }); + let point = poll_fn(|_| { + let mut io = self.io.borrow_mut(); + match io.chain_reply.take() { + Some(a) => { + io.pending = None; + Poll::Ready(a) + } + None => Poll::Pending, + } + }) + .await; + if point.height != height { + return Err(Error("chain_point")); + } + self.chain.borrow_mut().insert(height, point.hash); + Ok(()) + } +} +fn native_table(t: NativeTable) -> Table { + match t { + NativeTable::Directory => Table::Directory, + NativeTable::Pages(child) => Table::Pages { child }, + } +} +impl FilterSource for Driver { + async fn shard_map(&mut self) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Map).await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn filter(&mut self, shard_id: u64) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Filter { shard_id }).await?; + let n = b.len() as u64; + Ok((b, n)) + } +} +impl ShardTransport for Driver { + async fn init(&mut self) -> std::result::Result<(Vec, u64), BoxError> { + let b = self.send(Operation::Init).await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn manifest( + &mut self, + shard_id: u64, + revision: &str, + ) -> std::result::Result<(Vec, u64), BoxError> { + let b = self + .send(Operation::Manifest { + shard_id, + revision: revision.to_string(), + }) + .await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn setup( + &mut self, + shard_id: u64, + revision: &str, + table: NativeTable, + segment: u32, + ) -> std::result::Result<(Vec, u64), BoxError> { + let b = self + .send(Operation::Setup { + shard_id, + revision: revision.to_string(), + table: native_table(table), + segment, + }) + .await?; + let n = b.len() as u64; + Ok((b, n)) + } + async fn query( + &mut self, + shard_id: u64, + revision: &str, + table: NativeTable, + body: &[u8], + ) -> std::result::Result, BoxError> { + self.send(Operation::Query { + shard_id, + revision: revision.to_string(), + table: native_table(table), + body: STANDARD.encode(body), + }) + .await + } +} +struct Chain(Rc>>); +impl ChainView for Chain { + fn is_accepted(&self, h: u64, s: &str) -> Acceptance { + match self.0.borrow().get(&h) { + Some(held) if held == s => Acceptance::Accepted, + Some(_) => Acceptance::Rejected, + None => Acceptance::Unknown, + } + } + fn hash_at(&self, h: u64) -> Option { + self.0.borrow().get(&h).cloned() + } + fn tip(&self) -> Option { + self.0 + .borrow() + .iter() + .next_back() + .map(|(h, s)| crate::store::Anchor { + height: *h, + hash: s.clone(), + }) + } +} + +/// Exact held-checkpoint evidence from this pass's independently observed chain. +/// The owner must still validate its durable prior checkpoint lineage. This is +/// not a signature for arbitrary imported snapshots or caller-supplied headers. +/// +/// Holding the evidence prevents advancing, acknowledging or cancelling its pass. +/// ```compile_fail +/// use z_transparent_pir_v2::Pass; +/// fn cancel_while_held(pass: &mut Pass) { +/// let evidence = pass.accepted_rewind().unwrap(); +/// pass.cancel(); +/// drop(evidence); +/// } +/// ``` +/// ```compile_fail +/// use z_transparent_pir_v2::Pass; +/// fn acknowledge_while_held(pass: &mut Pass, id: &str) { +/// let evidence = pass.accepted_rewind().unwrap(); +/// pass.acknowledge(id).unwrap(); +/// drop(evidence); +/// } +/// ``` +/// ```compile_fail +/// use z_transparent_pir_v2::{AcceptedRewind, Scope, Checkpoint}; +/// fn forge<'a>(scope: &'a Scope, checkpoint: &'a Checkpoint) -> AcceptedRewind<'a> { +/// AcceptedRewind { scope, checkpoint } +/// } +/// ``` +pub struct AcceptedRewind<'pass> { + scope: &'pass Scope, + checkpoint: &'pass Checkpoint, +} +impl AcceptedRewind<'_> { + pub fn matches(&self, scope: &Scope, checkpoint: &Checkpoint) -> bool { + self.scope == scope && self.checkpoint == checkpoint + } +} + +pub struct Pass { + scope: Scope, + store: SharedStore, + driver: Driver, + // Prior evidence remains owned by the external durable checkpoint until + // its anchor is independently accepted or rejected on this target chain. + lineage_pending: Rc>, + future: Option>>, + previous: Option, + // Validated prior metadata, with large records/continuation owned by store. + prior: Option, + prior_baseline_matches: bool, + baseline: Vec, + state_baseline: Vec, + held: Option, + finished: Option, + cancelled: bool, + final_ack: bool, + terminal_without_checkpoint: bool, +} +fn encoded(store: &MemoryStore) -> Result> { + bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(store) + .map_err(|_| Error("checkpoint_encode")) +} +fn durable( + scope: &Scope, + state: &MemoryStore, + headers: &BTreeMap, + policy_limit: bool, +) -> Result> { + bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&Continuation { + version: 4, + source_id: scope.source_id.clone(), + state: state.clone(), + chain: ObservedChain { + anchor: scope.anchor.clone(), + genesis: headers.get(&0).cloned(), + headers: headers.clone(), + }, + policy_limit, + }) + .map_err(|_| Error("checkpoint_encode")) +} +fn hash(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn checkpoint_id( + source: &str, + scope: &str, + generation: u64, + anchor: &Anchor, + complete: bool, + bytes: &[u8], +) -> String { + let mut digest = Sha256::new(); + digest.update(source.as_bytes()); + digest.update(scope.as_bytes()); + digest.update(generation.to_le_bytes()); + digest.update(anchor.height.to_le_bytes()); + digest.update(anchor.hash.as_bytes()); + digest.update([u8::from(complete)]); + digest.update(bytes); + hex::encode(digest.finalize()) +} +impl Pass { + /// Borrow only the current held checkpoint's exact accepted rewind. No I/O. + pub fn accepted_rewind(&self) -> Result>> { + if self.cancelled { + return Err(Error("cancelled")); + } + if self.lineage_pending.get() { + return Err(Error("lineage_pending")); + } + let checkpoint = self.held.as_ref().ok_or(Error("checkpoint_missing"))?; + let Some(ancestor) = &checkpoint.rewind_to else { + return Ok(None); + }; + if self.driver.chain.borrow().get(&ancestor.height) != Some(&ancestor.hash) { + return Err(Error("rewind_not_observed")); + } + Ok(Some(AcceptedRewind { + scope: &self.scope, + checkpoint, + })) + } + + #[cfg(test)] + pub(crate) fn fixture_store(&self) -> SharedStore { + self.store.clone() + } + pub fn new(scope: Scope, checkpoint: Option, limits: Limits) -> Result { + if limits.max_chain_reads == 0 + || limits.max_chain_reads > 4096 + || limits.max_exchanges == 0 + || limits.max_exchanges > 4096 + || limits.max_scripts > 4096 + || !hash(&scope.source_id) + || !hash(&scope.scope_id) + || !hash(&scope.anchor.hash) + || scope.accepted_headers.len() > 4096 + || scope.scripts.len() > limits.max_scripts + || scope.scripts.is_empty() + { + return Err(Error("scope")); + } + let mut seen = std::collections::BTreeSet::new(); + if scope.scripts.iter().any(|s| !seen.insert(&s.script)) { + return Err(Error("duplicate_script")); + } + let mut restored_chain = None; + let mut restored_policy = false; + let mut memory = if let Some(cp) = &checkpoint { + if cp.anchor.height > scope.anchor.height { + return Err(Error("anchor_rewound")); + } + if cp.source_id != scope.source_id { + return Err(Error("source_changed")); + } + if cp.continuation.len() > 96 * 1024 * 1024 { + return Err(Error("checkpoint_limit")); + } + let bytes = STANDARD + .decode(&cp.continuation) + .map_err(|_| Error("checkpoint_encoding"))?; + let version = bytes.get(..4).ok_or(Error("checkpoint_decode"))?; + let (mut state, original) = match u32::from_le_bytes(version.try_into().unwrap()) { + 2 => { + let restored: LegacyContinuation = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(64 * 1024 * 1024) + .reject_trailing_bytes() + .deserialize(&bytes) + .map_err(|_| Error("checkpoint_decode"))?; + if restored.source_id != scope.source_id { + return Err(Error("checkpoint_source")); + } + let original = encoded(&restored.state)?; + (restored.state, original) + } + 3 | 4 => { + let options = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(MAX_CONTINUATION_BYTES as u64) + .reject_trailing_bytes(); + let restored: Continuation = if version == 3u32.to_le_bytes() { + let old: LegacyContinuationV3 = options + .deserialize(&bytes) + .map_err(|_| Error("checkpoint_decode"))?; + Continuation { + version: 4, + source_id: old.source_id, + state: old.state, + chain: old.chain, + policy_limit: false, + } + } else { + options + .deserialize(&bytes) + .map_err(|_| Error("checkpoint_decode"))? + }; + restored_policy = restored.policy_limit; + if restored.source_id != scope.source_id + || restored.chain.anchor != cp.anchor + || restored.chain.headers.len() > MAX_OBSERVED_POINTS + || restored + .chain + .headers + .iter() + .any(|(h, s)| *h > cp.anchor.height || !hash(s)) + || restored.chain.headers.get(&cp.anchor.height) != Some(&cp.anchor.hash) + || restored.chain.genesis.as_ref() != restored.chain.headers.get(&0) + { + return Err(Error("checkpoint_source")); + } + restored_chain = Some(restored.chain); + (restored.state, bytes) + } + _ => return Err(Error("checkpoint_source")), + }; + if checkpoint_id( + &cp.source_id, + &cp.scope_id, + cp.generation, + &cp.anchor, + cp.complete, + &original, + ) != cp.checkpoint_id + { + return Err(Error("checkpoint_id")); + } + state.rebuild_indexes(); + state + } else { + MemoryStore::new() + }; + let current_scripts = scope + .scripts + .iter() + .map(|s| hex::decode(&s.script).map_err(|_| Error("script"))) + .collect::>>()?; + memory.retain_scope(¤t_scripts); + let old_endpoints = retained_endpoints(&memory); + let unsupported_history = old_endpoints.is_err(); + let terminal_without_checkpoint = unsupported_history + || memory.events().map_err(|_| Error("store"))?.len() > MAX_RETAINED_EVENTS + || encoded(&memory)?.len() > MAX_STORE_BYTES; + memory.set_event_limit(MAX_RETAINED_EVENTS); + let old_endpoints = old_endpoints.unwrap_or_default(); + + let store = SharedStore { + inner: Rc::new(RefCell::new(memory)), + rewind: Rc::new(RefCell::new(None)), + policy_limit: Rc::new(Cell::new(restored_policy)), + }; + let mut headers = BTreeMap::new(); + for point in &scope.accepted_headers { + if !hash(&point.hash) || point.height > scope.anchor.height { + return Err(Error("chain")); + } + if let Some(old) = headers.insert(point.height, point.hash.clone()) { + if old != point.hash { + return Err(Error("chain")); + } + } + } + if headers.get(&scope.anchor.height) != Some(&scope.anchor.hash) { + return Err(Error("anchor_not_accepted")); + } + let driver = Driver { + io: Rc::new(RefCell::new(Io::default())), + limits: limits.clone(), + chain: Rc::new(RefCell::new( + headers + .iter() + .filter(|(h, _)| **h == 0 || **h == scope.anchor.height) + .map(|(h, s)| (*h, s.clone())) + .collect(), + )), + supplied: Rc::new(headers), + }; + // A moving tip may reuse older points only after independently checking + // that its previously accepted anchor still belongs to the selected chain. + let restored_chain = restored_chain.filter(|old| { + old.anchor.height <= scope.anchor.height + && old.genesis.is_some() + && old.genesis.as_ref() == driver.chain.borrow().get(&0) + }); + let lineage_pending = Rc::new(Cell::new( + restored_chain + .as_ref() + .is_some_and(|old| old.anchor != scope.anchor), + )); + if let Some(old) = &restored_chain { + if old.anchor == scope.anchor { + for (height, hash) in &old.headers { + if driver.supplied.get(height).is_some_and(|held| held != hash) { + return Err(Error("chain_point")); + } + driver.chain.borrow_mut().insert(*height, hash.clone()); + } + } + } + let mut filters = driver.clone(); + let mut transport = driver.clone(); + let owned = driver.clone(); + let chain = Chain(driver.chain.clone()); + let mut state = store.clone(); + let request = scope.clone(); + let lineage = lineage_pending.clone(); + let future = Box::pin(async move { + let (init, _) = transport.init().await.map_err(|_| Error("init"))?; + let parsed = + transparent_wallet::parse_init(&init).map_err(|_| Error("init_invalid"))?; + let geometry = ServiceGeometry { + schema: parsed.schema, + geometries: parsed + .geometries + .into_iter() + .map(|g| GeometryParams { + name: g.name, + directory_rows: g.directory_rows, + directory_row_bytes: g.directory_row_bytes, + directory_scheme: g.directory_scheme, + directory_scheme_dq44: g.directory_scheme_dq44, + directory_setup_seed: g.directory_setup_seed, + page_rows: g.page_rows, + page_row_bytes: g.page_row_bytes, + pages_scheme: g.pages_scheme, + pages_scheme_dq44: g.pages_scheme_dq44, + pages_setup_seed: g.pages_setup_seed, + }) + .collect(), + }; + let (raw, cost) = filters.shard_map().await.map_err(|_| Error("map"))?; + let map: transparent_filter::ShardMap = + serde_json::from_slice(&raw).map_err(|_| Error("map_invalid"))?; + map.check_shape().map_err(|_| Error("map_invalid"))?; + let genesis = match map.network.as_str() { + "main" | "mainnet" => transparent_filter::MAINNET_GENESIS_DISPLAY, + "test" | "testnet" => { + "05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38" + } + "regtest" => "029f11d80ef9765602235e1bc9727e3eb6ba20839319f761fee920d63401e327", + _ => { + return Ok(Status::Incomplete { + reason: "publication_profile".into(), + }) + } + }; + if map.genesis_hash != genesis + || map.profile != transparent_filter::RANGE_PROFILE_V2.name + { + return Ok(Status::Incomplete { + reason: "publication_profile".into(), + }); + } + owned.point(0, genesis).await?; + if !matches!(chain.is_accepted(0, genesis), Acceptance::Accepted) { + return Ok(Status::Incomplete { + reason: "chain_identity".into(), + }); + } + if map.shards.len() > MAX_MAP_SHARDS + || request + .scripts + .iter() + .any(|s| s.required_from < map.start_height) + { + return Ok(Status::Incomplete { + reason: "publication_incomplete".into(), + }); + } + let needed = needed_points( + map.shards.iter().flat_map(|entry| { + (entry.end_height <= request.anchor.height) + .then_some(entry.end_height) + .into_iter() + .chain( + entry + .start_height + .checked_sub(1) + .filter(|h| *h <= request.anchor.height), + ) + }), + &old_endpoints, + request.anchor.height, + restored_chain.as_ref().map(|old| old.anchor.height), + ); + owned.retain_points(&needed); + if let Some(old) = restored_chain { + if old.anchor != request.anchor && old.genesis.as_deref() == Some(genesis) { + owned.point(old.anchor.height, &old.anchor.hash).await?; + if matches!( + chain.is_accepted(old.anchor.height, &old.anchor.hash), + Acceptance::Accepted + ) { + for (height, hash) in + old.headers.into_iter().filter(|(h, _)| needed.contains(h)) + { + if owned + .chain + .borrow() + .get(&height) + .is_some_and(|held| held != &hash) + { + return Err(Error("chain_point")); + } + owned.chain.borrow_mut().insert(height, hash.clone()); + } + } + } + } + // No await separates the accepted-cache merge / rejected-cache + // discard from this decision. Only now may a replacement checkpoint + // bind the journal and observations to the new target. + lineage.set(false); + for e in &map.shards { + if e.end_height <= request.anchor.height { + owned.point(e.end_height, &e.terminal_block_hash).await?; + } + if e.start_height > 0 && e.start_height - 1 <= request.anchor.height { + owned + .point(e.start_height - 1, &e.parent_block_hash) + .await?; + } + } + for e in state.scripts().map_err(|_| Error("store"))? { + for range in state.coverage(&e.script).map_err(|_| Error("store"))? { + owned + .point(range.end_height, &range.terminal_block_hash) + .await?; + } + } + let scripts = request + .scripts + .iter() + .map(|s| { + Ok(ScriptEntry { + script: hex::decode(&s.script).map_err(|_| Error("script"))?, + origin: ScriptOrigin::Derived, + required_from: s.required_from, + }) + }) + .collect::>>()?; + if scripts.iter().any(|s| { + s.script.is_empty() + || s.script.len() > 40 + || s.required_from > request.anchor.height + }) { + return Err(Error("script")); + } + let mut scripts = StaticScripts(scripts); + let target = crate::store::Anchor { + height: request.anchor.height, + hash: request.anchor.hash, + }; + match sync::sync_into( + &mut state, + &map, + cost, + &geometry, + &chain, + &mut scripts, + &mut filters, + &mut transport, + &WorkLimits { + max_queries: Some(limits.max_queries), + max_private_bytes: Some(limits.max_private_bytes), + }, + &target, + ) + .await + { + Ok(report) => Ok(match report.completion { + Completion::Complete => Status::Complete, + Completion::Incomplete { reason, .. } => Status::Incomplete { + reason: reason_code(reason).into(), + }, + }), + Err(_) => Ok(Status::Incomplete { + reason: "recovery_failed".into(), + }), + } + }); + let baseline = durable( + &scope, + &store.inner.borrow(), + &driver.chain.borrow(), + store.policy_limit.get(), + )?; + let state_baseline = encoded(&store.inner.borrow())?; + let prior_baseline_matches = checkpoint.as_ref().is_some_and(|cp| { + cp.scope_id == scope.scope_id + && cp.generation == scope.generation + && cp.anchor == scope.anchor + && STANDARD.decode(&cp.continuation).ok().as_deref() == Some(baseline.as_slice()) + }); + let previous = checkpoint.as_ref().map(|cp| cp.checkpoint_id.clone()); + let prior = checkpoint.map(|mut cp| { + cp.records = Vec::new(); + cp.continuation = String::new(); + cp + }); + Ok(Self { + scope, + store, + driver, + lineage_pending, + future: (!terminal_without_checkpoint && !restored_policy).then_some(future), + previous, + prior, + prior_baseline_matches, + baseline, + state_baseline, + held: None, + finished: (terminal_without_checkpoint || restored_policy).then(|| { + Status::Incomplete { + reason: "history_metadata_limit".into(), + } + }), + cancelled: false, + final_ack: terminal_without_checkpoint || restored_policy, + terminal_without_checkpoint, + }) + } + fn stop_for_policy(&mut self) { + self.future = None; + self.finished = Some(Status::Incomplete { + reason: "history_metadata_limit".into(), + }); + let mut io = self.driver.io.borrow_mut(); + io.pending = None; + io.reply = None; + io.chain_reply = None; + } + pub fn advance(&mut self) -> Result { + if self.cancelled { + return Ok(Step::Stopped { + status: Status::Cancelled, + }); + } + if self.terminal_without_checkpoint { + return Ok(Step::Stopped { + status: Status::Incomplete { + reason: "history_metadata_limit".into(), + }, + }); + } + if self.store.policy_limit.get() { + self.stop_for_policy(); + } + if let Some(cp) = &self.held { + return Ok(Step::Checkpoint { + checkpoint: cp.clone(), + }); + } + if self.finished.is_none() { + if let Some(future) = self.future.as_mut() { + let mut cx = Context::from_waker(noop_waker_ref()); + if let Poll::Ready(result) = future.as_mut().poll(&mut cx) { + self.finished = Some(if let Some(reason) = self.driver.io.borrow().stop { + Status::Incomplete { + reason: reason.into(), + } + } else { + result.unwrap_or_else(|_| Status::Incomplete { + reason: "unavailable".into(), + }) + }); + self.future = None; + } + } + } + if self.store.policy_limit.get() { + self.stop_for_policy(); + } + if self.lineage_pending.get() && self.finished.is_some() { + // The caller keeps its exact acknowledged prior checkpoint. An Init, + // Map or old-anchor failure/deadline is not an invalidation decision. + return Ok(Step::Stopped { + status: self.finished.clone().unwrap(), + }); + } + let state_changed = encoded(&self.store.inner.borrow())? != self.state_baseline; + if state_changed && retained_endpoints(&self.store.inner.borrow()).is_err() { + // Preserve every accepted fact and outstanding cover obligation. + self.store.policy_limit.set(true); + self.stop_for_policy(); + } + if state_changed || self.finished.is_some() { + let bytes = durable( + &self.scope, + &self.store.inner.borrow(), + &self.driver.chain.borrow(), + self.store.policy_limit.get(), + )?; + if self.prior_baseline_matches + && bytes == self.baseline + && matches!(self.finished, Some(Status::Incomplete { .. })) + { + return Ok(Step::Stopped { + status: self.finished.clone().unwrap(), + }); + } + // A final status checkpoint is emitted once, then removed by acknowledge. + if self.finished.is_some() + && self.future.is_none() + && self.baseline == bytes + && self.final_ack + { + return Ok(Step::Stopped { + status: self.finished.clone().unwrap(), + }); + } + let cp = self.checkpoint(&bytes)?; + self.held = Some(cp.clone()); + return Ok(Step::Checkpoint { checkpoint: cp }); + } + match self.driver.io.borrow().pending.clone() { + Some(Pending::Exchange { id, operation }) => Ok(Step::Exchange { + request_id: id, + operation, + }), + Some(Pending::Chain { height, hash }) => Ok(Step::NeedChain { height, hash }), + None => Err(Error("no_progress")), + } + } + fn checkpoint(&self, bytes: &[u8]) -> Result { + if self.lineage_pending.get() { + return Err(Error("lineage_pending")); + } + let state = self.store.inner.borrow(); + let complete = self.finished == Some(Status::Complete); + let records = state + .events() + .map_err(|_| Error("store"))? + .into_iter() + .map(|e| Record { + script: hex::encode(e.script), + event: hex::encode(e.event.to_bytes()), + shard_id: e.shard_id, + revision: e.revision_digest, + }) + .collect(); + if state.events().map_err(|_| Error("store"))?.len() > MAX_RETAINED_EVENTS { + return Err(Error("event_limit")); + } + let mut coverage = Vec::new(); + let mut through = if state.scripts().map_err(|_| Error("store"))?.is_empty() { + 0 + } else { + self.scope.anchor.height + }; + for script in state.scripts().map_err(|_| Error("store"))? { + let ranges = state.coverage(&script.script).map_err(|_| Error("store"))?; + let mut end: Option = None; + let mut revision = String::new(); + for r in ranges { + let next = end + .map(|height| height.saturating_add(1)) + .unwrap_or(script.required_from); + if r.start_height <= next && r.end_height >= next { + end = Some(r.end_height); + revision = r.revision_digest; + } + } + through = through.min(end.unwrap_or(script.required_from.saturating_sub(1))); + if let Some(end) = end { + coverage.push(Coverage { + script: hex::encode(script.script), + required_from: script.required_from, + through: end, + revision, + }); + } + } + if complete + && (coverage.len() != self.scope.scripts.len() + || coverage + .iter() + .any(|range| range.through < self.scope.anchor.height)) + { + return Err(Error("coverage_incomplete")); + } + if bytes.len() > MAX_CONTINUATION_BYTES { + return Err(Error("checkpoint_limit")); + } + let continuation = STANDARD.encode(bytes); + let id = checkpoint_id( + &self.scope.source_id, + &self.scope.scope_id, + self.scope.generation, + &self.scope.anchor, + complete, + bytes, + ); + let checkpoint = Checkpoint { + source_id: self.scope.source_id.clone(), + generation: self.scope.generation, + scope_id: self.scope.scope_id.clone(), + checkpoint_id: id.clone(), + previous_checkpoint_id: if self.previous.as_ref() == Some(&id) { + self.prior + .as_ref() + .and_then(|cp| cp.previous_checkpoint_id.clone()) + } else { + self.previous.clone() + }, + anchor: self.scope.anchor.clone(), + covered_through: through, + complete, + pending_pages: state.pending().map_err(|_| Error("store"))?.len() as u64, + unresolved_spends: state + .ledger() + .map_err(|_| Error("store"))? + .unresolved() + .len() as u64, + records, + coverage, + rewind_to: if self.previous.as_ref() == Some(&id) { + self.prior.as_ref().and_then(|cp| cp.rewind_to.clone()) + } else { + self.store.rewind.borrow().clone().map(|a| Anchor { + height: a.height, + hash: a.hash, + }) + }, + continuation, + }; + if self.previous.as_ref() == Some(&id) { + let mut prior = self.prior.clone().ok_or(Error("checkpoint_missing"))?; + prior.records = checkpoint.records.clone(); + prior.continuation = checkpoint.continuation.clone(); + if prior != checkpoint { + return Err(Error("checkpoint_conflict")); + } + Ok(prior) + } else { + Ok(checkpoint) + } + } + pub fn accept(&mut self, request_id: u64, reply: Reply) -> Result<()> { + if self.held.is_some() || self.cancelled { + return Err(Error("not_waiting")); + } + let mut io = self.driver.io.borrow_mut(); + if !matches!(io.pending,Some(Pending::Exchange{id,..})if id==request_id) + || io.reply.is_some() + { + return Err(Error("request_id")); + } + let bytes = serde_json::to_vec(&reply) + .map_err(|_| Error("reply"))? + .len() as u64; + io.download = io.download.saturating_add(bytes); + if io.download > self.driver.limits.max_download_bytes { + io.stop = Some("budget"); + io.reply = Some(Reply::Unavailable) + } else { + io.reply = Some(reply) + } + Ok(()) + } + pub fn accept_chain(&mut self, height: u64, hash: String) -> Result<()> { + if !self.held.is_none() || !crate::pass::hash(&hash) { + return Err(Error("chain_point")); + } + let mut io = self.driver.io.borrow_mut(); + if !matches!(io.pending,Some(Pending::Chain{height:h,..})if h==height) + || io.chain_reply.is_some() + { + return Err(Error("chain_point")); + } + io.chain_reply = Some(Anchor { height, hash }); + Ok(()) + } + pub fn acknowledge(&mut self, id: &str) -> Result<()> { + let cp = self.held.as_ref().ok_or(Error("checkpoint_missing"))?; + if cp.checkpoint_id != id { + return Err(Error("checkpoint_id")); + } + self.baseline = durable( + &self.scope, + &self.store.inner.borrow(), + &self.driver.chain.borrow(), + self.store.policy_limit.get(), + )?; + self.state_baseline = encoded(&self.store.inner.borrow())?; + self.prior_baseline_matches = true; + self.previous = Some(id.into()); + self.final_ack = self.finished.is_some(); + self.store.rewind.borrow_mut().take(); + let mut cp = self.held.take().ok_or(Error("checkpoint_missing"))?; + cp.records = Vec::new(); + cp.continuation = String::new(); + self.prior = Some(cp); + Ok(()) + } + /// End an active resource-limited pass without discarding validated logical + /// progress. The caller must still durably acknowledge any yielded checkpoint. + /// This must never be used for user cancellation or a retired wallet session. + pub fn finish_budget(&mut self) -> Result<()> { + if self.cancelled { + return Err(Error("cancelled")); + } + if self.final_ack && self.finished.is_some() { + return Ok(()); + } + self.future = None; + if self.finished.is_none() { + self.finished = Some(Status::Incomplete { + reason: "budget".into(), + }); + } + let mut io = self.driver.io.borrow_mut(); + io.pending = None; + io.reply = None; + io.chain_reply = None; + if self.lineage_pending.get() { + self.final_ack = true; + return Ok(()); + } + // With no new accepted work, the preceding durable checkpoint already + // represents this pass; do not emit a conflicting same-ID replay. + if self.held.is_none() { + self.final_ack = self.prior_baseline_matches + && self.baseline + == durable( + &self.scope, + &self.store.inner.borrow(), + &self.driver.chain.borrow(), + self.store.policy_limit.get(), + )?; + } + Ok(()) + } + pub fn cancel(&mut self) { + self.cancelled = true; + self.future = None; + self.held = None; + self.driver.io.borrow_mut().pending = None; + } +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod long_fork_tests; + +#[cfg(test)] +mod accepted_rewind_tests; diff --git a/crates/z-transparent-pir-v2/src/pass/accepted_rewind_tests.rs b/crates/z-transparent-pir-v2/src/pass/accepted_rewind_tests.rs new file mode 100644 index 0000000..239c1fa --- /dev/null +++ b/crates/z-transparent-pir-v2/src/pass/accepted_rewind_tests.rs @@ -0,0 +1,103 @@ +use super::*; + +fn held() -> Pass { + let mut pass = Pass::new(super::tests::scope(), None, Limits::default()).unwrap(); + let mut checkpoint = super::tests::snapshot(&pass); + checkpoint.rewind_to = Some(Anchor { + height: 20, + hash: "b".repeat(64), + }); + pass.driver.chain.borrow_mut().insert(20, "b".repeat(64)); + pass.held = Some(checkpoint); + pass +} + +#[test] +fn accepted_rewind_binds_full_scope_and_full_checkpoint() { + let pass = held(); + let scope = pass.scope.clone(); + let checkpoint = pass.held.as_ref().unwrap().clone(); + let witness = pass.accepted_rewind().unwrap().unwrap(); + assert!(witness.matches(&scope, &checkpoint)); + let mut scopes = Vec::new(); + let mut changed = scope.clone(); + changed.source_id = "c".repeat(64); + scopes.push(changed); + let mut changed = scope.clone(); + changed.scope_id = "c".repeat(64); + scopes.push(changed); + let mut changed = scope.clone(); + changed.generation += 1; + scopes.push(changed); + let mut changed = scope.clone(); + changed.anchor.hash = "c".repeat(64); + scopes.push(changed); + let mut changed = scope.clone(); + changed.scripts[0].required_from += 1; + scopes.push(changed); + let mut changed = scope.clone(); + changed.scripts[0].script.push_str("00"); + scopes.push(changed); + let mut changed = scope.clone(); + changed.accepted_headers.push(Anchor { + height: 1, + hash: "c".repeat(64), + }); + scopes.push(changed); + for changed in scopes { + assert!(!witness.matches(&changed, &checkpoint)); + } + let mut checkpoints = Vec::new(); + let mut changed = checkpoint.clone(); + changed.previous_checkpoint_id = Some("c".repeat(64)); + checkpoints.push(changed); + let mut changed = checkpoint.clone(); + changed.continuation.push('A'); + checkpoints.push(changed); + let mut changed = checkpoint.clone(); + changed.complete = !changed.complete; + checkpoints.push(changed); + let mut changed = checkpoint.clone(); + changed.rewind_to.as_mut().unwrap().hash = "c".repeat(64); + checkpoints.push(changed); + let mut changed = checkpoint.clone(); + changed.records.push(Record { + script: scope.scripts[0].script.clone(), + event: "00".into(), + shard_id: 0, + revision: "c".repeat(64), + }); + checkpoints.push(changed); + for changed in checkpoints { + assert_eq!(changed.checkpoint_id, checkpoint.checkpoint_id); + assert!(!witness.matches(&scope, &changed)); + } + // Identical context is the same authority; the owner still applies its + // prior-ID/digest and idempotent replay checks. A different held CP is not. + let other = held(); + assert!(witness.matches(&other.scope, other.held.as_ref().unwrap())); +} + +#[test] +fn accepted_rewind_requires_live_resolved_matching_observation() { + let mut pass = held(); + pass.driver.chain.borrow_mut().remove(&20); + assert_eq!( + pass.accepted_rewind().err().unwrap().0, + "rewind_not_observed" + ); + pass.driver.chain.borrow_mut().insert(20, "c".repeat(64)); + assert_eq!( + pass.accepted_rewind().err().unwrap().0, + "rewind_not_observed" + ); + pass.driver.chain.borrow_mut().insert(20, "b".repeat(64)); + pass.lineage_pending.set(true); + assert_eq!(pass.accepted_rewind().err().unwrap().0, "lineage_pending"); + pass.lineage_pending.set(false); + assert!(pass.accepted_rewind().unwrap().is_some()); + pass.held.as_mut().unwrap().rewind_to = None; + assert!(pass.accepted_rewind().unwrap().is_none()); + pass.cancel(); + assert_eq!(pass.accepted_rewind().err().unwrap().0, "cancelled"); +} diff --git a/crates/z-transparent-pir-v2/src/pass/long_fork_tests.rs b/crates/z-transparent-pir-v2/src/pass/long_fork_tests.rs new file mode 100644 index 0000000..ffb9be4 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/pass/long_fork_tests.rs @@ -0,0 +1,481 @@ +#[path = "../../tests/fixture.rs"] +mod fixture; +use super::*; +use axum::http::Request; +use base64::{engine::general_purpose::STANDARD, Engine}; +use fixture::{publish, synthetic, InProcess, ShardSpec, SEAL}; +use transparent_events::{ReceiveEvent, SpendEvent, TransparentEvent, Txid}; +use transparent_filter::ScriptBytes; + +fn script(tag: u8) -> ScriptBytes { + let mut s = vec![0x76, 0xa9, 0x14]; + s.extend([tag; 20]); + s.extend([0x88, 0xac]); + ScriptBytes::new(s) +} +fn events() -> Vec<(ScriptBytes, TransparentEvent)> { + vec![ + ( + script(1), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 101, + txid: Txid([7; 32]), + transaction_index: 0, + output_index: 0, + value: 50000, + coinbase: false, + }), + ), + ( + script(1), + TransparentEvent::Spend(SpendEvent { + metadata: None, + height: 102, + spending_txid: Txid([8; 32]), + transaction_index: 0, + input_index: 0, + spent_txid: Txid([7; 32]), + spent_output_index: 0, + }), + ), + ( + script(2), + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height: 103, + txid: Txid([9; 32]), + transaction_index: 0, + output_index: 0, + value: 17000, + coinbase: false, + }), + ), + ] +} +fn scope() -> Scope { + Scope { + source_id: "1".repeat(64), + generation: 1, + scope_id: "2".repeat(64), + anchor: Anchor { + height: 110, + hash: synthetic(110).to_display_hex(), + }, + scripts: (1..=3) + .map(|i| Script { + script: hex::encode(script(i).as_slice()), + required_from: 100, + }) + .collect(), + accepted_headers: std::iter::once(Anchor { + height: 0, + hash: transparent_filter::MAINNET_GENESIS_DISPLAY.into(), + }) + .chain((99..=110).map(|h| Anchor { + height: h, + hash: synthetic(h).to_display_hex(), + })) + .collect(), + } +} +fn path(table: Table) -> String { + match table { + Table::Directory => "directory".into(), + Table::Pages { child } => format!("pages-{child}"), + } +} +fn answer(server: &InProcess, operation: Operation) -> Reply { + let (method, url, body) = match operation { + Operation::Init => ("GET", "/v2/shards/init".into(), vec![]), + Operation::Map => ("GET", "/v2/filters/shards".into(), vec![]), + Operation::Filter { shard_id } => ( + "GET", + format!("/v2/filters/shards/{shard_id}/filter"), + vec![], + ), + Operation::Manifest { shard_id, revision } => ( + "GET", + format!("/v2/shards/{shard_id}/revisions/{revision}/manifest"), + vec![], + ), + Operation::Setup { + shard_id, + revision, + table, + segment, + } => ( + "GET", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/setup/{}/{segment}", + path(table) + ), + vec![], + ), + Operation::Query { + shard_id, + revision, + table, + body, + } => ( + "POST", + format!( + "/v2/shards/{shard_id}/revisions/{revision}/query/{}", + path(table) + ), + STANDARD.decode(body).unwrap(), + ), + }; + let response = server.answer( + Request::builder() + .method(method) + .uri(url) + .header("content-type", "application/octet-stream") + .body(body) + .unwrap(), + ); + assert_eq!( + response.status(), + 200, + "fixture reply {}", + String::from_utf8_lossy(response.body()) + ); + Reply::Data { + body: STANDARD.encode(response.body()), + } +} + +#[test] +fn long_fork_observations_survive_budget_and_elapsed_cold_restarts() { + let old_dir = tempfile::tempdir().unwrap(); + let shards: Vec<_> = std::iter::once(ShardSpec::first(100, 110, true, events())) + .chain((111..=245).map(|h| ShardSpec::first(h, h, true, vec![]))) + .collect(); + publish(old_dir.path(), &shards, SEAL, synthetic, vec![]); + let old = InProcess::serve(old_dir.path()); + let mut request = scope(); + request.anchor = Anchor { + height: 245, + hash: synthetic(245).to_display_hex(), + }; + request + .accepted_headers + .extend((111..=245).map(|height| Anchor { + height, + hash: synthetic(height).to_display_hex(), + })); + let mut previous = None; + for _ in 0..12 { + let mut p = Pass::new(request.clone(), previous.clone(), Limits::default()).unwrap(); + for _ in 0..4000 { + match p.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => p.accept(request_id, answer(&old, operation)).unwrap(), + Step::NeedChain { .. } => { + panic!("old fixture independently supplies its accepted chain") + } + Step::Checkpoint { mut checkpoint } => { + p.acknowledge(&checkpoint.checkpoint_id).unwrap(); + checkpoint.records.clear(); + previous = Some(checkpoint); + } + Step::Stopped { .. } => break, + } + } + if previous.as_ref().is_some_and(|cp: &Checkpoint| cp.complete) { + break; + } + } + let old_cp = previous.unwrap(); + assert!(old_cp.complete, "old genuine acquisition must finish"); + drop(old); + let fork = |h| { + if h >= 105 { + synthetic(h + 1000) + } else { + synthetic(h) + } + }; + let fork_dir = tempfile::tempdir().unwrap(); + publish( + fork_dir.path(), + &[ShardSpec::first(100, 245, true, events())], + SEAL, + fork, + vec![], + ); + let server = InProcess::serve(fork_dir.path()); + request.anchor.hash = fork(245).to_display_hex(); + request.accepted_headers.retain(|a| a.height == 0); + request.accepted_headers.push(request.anchor.clone()); + let advanced_dir = tempfile::tempdir().unwrap(); + publish( + advanced_dir.path(), + &[ShardSpec::first(100, 247, true, events())], + SEAL, + fork, + vec![], + ); + let advanced_server = InProcess::serve(advanced_dir.path()); + let second_dir = tempfile::tempdir().unwrap(); + publish( + second_dir.path(), + &[ShardSpec::first(100, 247, true, events())], + SEAL, + |h| { + if h >= 105 { + synthetic(h + 2000) + } else { + synthetic(h) + } + }, + vec![], + ); + let second_server = InProcess::serve(second_dir.path()); + for scenario in ["stable", "advancing_reorg"] { + let mut previous = Some(old_cp.clone()); + let mut previously_read = std::collections::BTreeSet::new(); + let mut complete = false; + for round in 0..12 { + let epoch = usize::from(scenario == "advancing_reorg" && round >= 3); + let target = if scenario == "stable" { + 245 + } else { + (245 + round).min(247) + }; + let actual = |height| { + if height >= 105 { + synthetic(height + if epoch == 0 { 1000 } else { 2000 }) + } else { + synthetic(height) + } + }; + request.anchor = Anchor { + height: target, + hash: actual(target).to_display_hex(), + }; + request.accepted_headers.retain(|a| a.height == 0); + request.accepted_headers.push(request.anchor.clone()); + let current_server = if scenario == "stable" { + &server + } else if epoch == 0 { + &advanced_server + } else { + &second_server + }; + let mut p = Pass::new(request.clone(), previous.clone(), Limits::default()).unwrap(); + let initial_live = p.driver.chain.borrow().len(); + let mut reads = 0; + let mut mismatches = 0; + let mut refetched = 0; + let mut walker = false; + let mut rewound = false; + for _ in 0..4000 { + match p.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + walker |= !matches!(operation, Operation::Init | Operation::Map); + p.accept(request_id, answer(current_server, operation)) + .unwrap(); + } + Step::NeedChain { height, hash } => { + if round == 0 && reads == 12 { + p.finish_budget().unwrap(); + continue; + } + reads += 1; + let actual = actual(height).to_display_hex(); + mismatches += usize::from(actual != hash); + refetched += usize::from(!previously_read.insert(height)); + p.accept_chain(height, actual).unwrap(); + } + Step::Checkpoint { mut checkpoint } => { + rewound |= checkpoint.rewind_to.is_some(); + walker |= checkpoint.rewind_to.is_some(); + if !walker { + assert_eq!( + checkpoint.records.len(), + 3, + "header-only checkpoints preserve genuine old receive/spend facts" + ); + assert!(checkpoint.rewind_to.is_none()); + } + p.acknowledge(&checkpoint.checkpoint_id).unwrap(); + checkpoint.records.clear(); + previous = Some(checkpoint); + } + Step::Stopped { status } => { + complete = status == Status::Complete; + break; + } + } + } + if scenario == "stable" && round == 0 { + interrupted_advance_keeps_prior_evidence( + &advanced_server, + &request, + previous.as_ref().unwrap(), + ); + } + eprintln!( + "LONG_FORK_CENSUS {}", + serde_json::json!({"scenario":scenario,"fork_epoch":epoch,"round":round,"exit":if round==0{"elapsed"}else{"chain_budget"},"reads":reads,"mismatches":mismatches,"refetched":refetched,"initial_live":initial_live,"live":p.driver.chain.borrow().len(),"durable":p.driver.chain.borrow().len(),"walker_reached":walker,"rewind":rewound,"complete":complete}) + ); + if complete { + break; + } + } + assert!( + complete, + "bounded cold passes must reach verified long-fork rollback: {scenario}" + ); + } +} + +fn observed_count(cp: &Checkpoint) -> usize { + let decoded: Continuation = bincode::DefaultOptions::new() + .with_fixint_encoding() + .deserialize(&STANDARD.decode(&cp.continuation).unwrap()) + .unwrap(); + decoded.chain.headers.len() +} +fn interrupted_advance_keeps_prior_evidence( + server: &InProcess, + prior_scope: &Scope, + prior: &Checkpoint, +) { + let prior_count = observed_count(prior); + assert!( + prior_count > 10, + "genuine checkpoint must contain a nontrivial independently observed prefix" + ); + let mut scope = prior_scope.clone(); + scope.anchor = Anchor { + height: 246, + hash: synthetic(1246).to_display_hex(), + }; + scope.accepted_headers.retain(|p| p.height == 0); + scope.accepted_headers.push(scope.anchor.clone()); + for mode in [ + "init_error", + "map_error", + "before_init_deadline", + "init_deadline", + "map_deadline", + "anchor_deadline", + "unpolled_anchor_deadline", + "accepted_anchor_budget", + "rejected_anchor_budget", + ] { + let decided = mode == "accepted_anchor_budget" || mode == "rejected_anchor_budget"; + let mut selected = scope.clone(); + if mode == "rejected_anchor_budget" { + selected.anchor.hash = synthetic(2246).to_display_hex(); + selected.accepted_headers.pop(); + selected.accepted_headers.push(selected.anchor.clone()); + } + let mut limits = Limits::default(); + if decided { + limits.max_chain_reads = 1; + } + let mut p = Pass::new(selected.clone(), Some(prior.clone()), limits).unwrap(); + let state_before = encoded(&p.store.inner.borrow()).unwrap(); + let mut persisted = prior.clone(); + let mut checkpoints = 0; + if mode == "before_init_deadline" { + p.finish_budget().unwrap(); + } + for _ in 0..20 { + match p.advance().unwrap() { + Step::Exchange { + request_id, + operation, + } => { + let at_init = matches!(operation, Operation::Init); + let at_map = matches!(operation, Operation::Map); + assert!(at_init || at_map, "no query may precede lineage decision"); + if (at_init && mode == "init_deadline") || (at_map && mode == "map_deadline") { + p.finish_budget().unwrap(); + } else if (at_init && mode == "init_error") || (at_map && mode == "map_error") { + p.accept(request_id, Reply::Unavailable).unwrap(); + } else { + p.accept(request_id, answer(server, operation)).unwrap(); + } + } + Step::NeedChain { height, hash } => { + assert_eq!(height, prior.anchor.height); + assert_eq!(hash, prior.anchor.hash); + assert!( + decided || mode == "anchor_deadline" || mode == "unpolled_anchor_deadline" + ); + if decided { + let actual = if mode == "rejected_anchor_budget" { + synthetic(2245).to_display_hex() + } else { + hash + }; + p.accept_chain(height, actual).unwrap(); + } else { + if mode == "unpolled_anchor_deadline" { + p.accept_chain(height, hash).unwrap(); + } + p.finish_budget().unwrap(); + } + } + Step::Checkpoint { checkpoint } => { + checkpoints += 1; + p.acknowledge(&checkpoint.checkpoint_id).unwrap(); + persisted = checkpoint; + } + Step::Stopped { status } => { + assert!(matches!(status, Status::Incomplete { .. })); + break; + } + } + } + eprintln!( + "LINEAGE_CENSUS {}", + serde_json::json!({"mode":mode,"prior_observed":prior_count,"persisted_observed":observed_count(&persisted),"replacement_checkpoints":checkpoints,"facts_unchanged":encoded(&p.store.inner.borrow()).unwrap()==state_before,"continuation_unchanged":persisted.continuation==prior.continuation}) + ); + if decided { + assert_eq!( + checkpoints, 1, + "decided lineage must durably publish its atomic cache decision" + ); + assert_ne!(persisted.anchor, prior.anchor); + assert_eq!( + persisted.records.len(), + 3, + "header-only decision preserves genuine facts" + ); + assert!(persisted.rewind_to.is_none()); + assert_eq!( + p.driver.chain.borrow().contains_key(&110), + mode == "accepted_anchor_budget", + "only accepted lineage inherits older observations" + ); + let cold = Pass::new(selected, Some(persisted.clone()), Limits::default()).unwrap(); + assert_eq!( + cold.driver.chain.borrow().contains_key(&110), + mode == "accepted_anchor_budget" + ); + } else { + assert_eq!( + checkpoints, 0, + "an undecided lineage must not replace its prior evidence: {mode}" + ); + assert!( + persisted.continuation == prior.continuation, + "prior opaque continuation must survive exactly" + ); + } + assert!( + encoded(&p.store.inner.borrow()).unwrap() == state_before, + "facts/debt cannot change before lineage decision" + ); + } +} diff --git a/crates/z-transparent-pir-v2/src/pass/tests.rs b/crates/z-transparent-pir-v2/src/pass/tests.rs new file mode 100644 index 0000000..4347323 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/pass/tests.rs @@ -0,0 +1,937 @@ +use super::*; +use futures::Future; +pub(super) fn scope() -> Scope { + Scope { + source_id: "1".repeat(64), + generation: 1, + scope_id: "2".repeat(64), + anchor: Anchor { + height: 100, + hash: "a".repeat(64), + }, + scripts: vec![Script { + script: "76a914000000000000000000000000000000000000000088ac".into(), + required_from: 0, + }], + accepted_headers: vec![ + Anchor { + height: 0, + hash: transparent_filter::MAINNET_GENESIS_DISPLAY.into(), + }, + Anchor { + height: 100, + hash: "a".repeat(64), + }, + ], + } +} +pub(super) fn snapshot(p: &Pass) -> Checkpoint { + let bytes = durable( + &p.scope, + &p.store.inner.borrow(), + &p.driver.chain.borrow(), + p.store.policy_limit.get(), + ) + .unwrap(); + p.checkpoint(&bytes).unwrap() +} +#[test] +fn version_two_migrates_only_after_its_original_digest_is_verified() { + let p = Pass::new(scope(), None, Limits::default()).unwrap(); + let script = hex::decode(&scope().scripts[0].script).unwrap(); + p.store + .inner + .borrow_mut() + .add_scripts(&[ScriptEntry { + script: script.clone(), + origin: ScriptOrigin::Derived, + required_from: 0, + }]) + .unwrap(); + p.store + .inner + .borrow_mut() + .commit_shard(crate::store::ShardCommit { + end_height: 100, + terminal_block_hash: "a".repeat(64), + revision_digest: "b".repeat(64), + failed_entries: vec![script.clone()], + pending_upsert: vec![crate::store::PendingPages { + id: None, + shard_id: 0, + revision_digest: "b".repeat(64), + script: script.clone(), + cursor: crate::store::RunCursor { + child: 0, + row: 0, + older_runs: None, + }, + carried: vec![], + fragments: vec![], + dummy_debt: Some((2, 3)), + attempts: 1, + target_anchor: Some(crate::store::Anchor { + height: 100, + hash: "a".repeat(64), + }), + }], + ..Default::default() + }) + .unwrap(); + let mut cp = snapshot(&p); + let state = p.store.inner.borrow().clone(); + cp.continuation = STANDARD.encode( + bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&LegacyContinuation { + version: 2, + source_id: cp.source_id.clone(), + state: state.clone(), + }) + .unwrap(), + ); + cp.checkpoint_id = checkpoint_id( + &cp.source_id, + &cp.scope_id, + cp.generation, + &cp.anchor, + false, + &encoded(&state).unwrap(), + ); + let restored = Pass::new(scope(), Some(cp.clone()), Limits::default()).unwrap(); + assert_eq!( + restored.store.inner.borrow().pending().unwrap()[0].dummy_debt, + Some((2, 3)) + ); + assert_eq!( + restored + .store + .inner + .borrow() + .entry_failures(0, &script) + .unwrap(), + 1 + ); + let upgraded = snapshot(&restored); + assert_eq!( + &STANDARD.decode(&upgraded.continuation).unwrap()[..4], + &4u32.to_le_bytes() + ); + let mut moved = scope(); + moved.anchor = Anchor { + height: 101, + hash: "c".repeat(64), + }; + moved.accepted_headers.pop(); + moved.accepted_headers.push(moved.anchor.clone()); + let mut interrupted = Pass::new(moved, Some(upgraded), Limits::default()).unwrap(); + interrupted.finish_budget().unwrap(); + assert!( + matches!(interrupted.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="budget") + ); + assert_eq!( + interrupted.store.inner.borrow().pending().unwrap()[0].dummy_debt, + Some((2, 3)) + ); + assert_eq!( + interrupted + .store + .inner + .borrow() + .entry_failures(0, &script) + .unwrap(), + 1 + ); + interrupted.cancel(); + assert!(interrupted.finish_budget().is_err()); + cp.checkpoint_id = "f".repeat(64); + assert!(Pass::new(scope(), Some(cp), Limits::default()).is_err()); +} +#[test] +fn observed_cache_is_bound_and_rejects_tampering_and_oversized_maps() { + let p = Pass::new(scope(), None, Limits::default()).unwrap(); + p.driver.chain.borrow_mut().insert(50, "b".repeat(64)); + let cp = snapshot(&p); + let same = Pass::new(scope(), Some(cp.clone()), Limits::default()).unwrap(); + assert!(same.driver.chain.borrow().contains_key(&50)); + let mut changed = scope(); + changed.anchor.height += 1; + changed.anchor.hash = "c".repeat(64); + changed.accepted_headers.pop(); + changed.accepted_headers.push(changed.anchor.clone()); + let moving = Pass::new(changed, Some(cp.clone()), Limits::default()).unwrap(); + assert!( + !moving.driver.chain.borrow().contains_key(&50), + "advancing anchor must first be revalidated" + ); + let mut changed = scope(); + changed.source_id = "d".repeat(64); + assert!(Pass::new(changed, Some(cp.clone()), Limits::default()).is_err()); + let mut changed = scope(); + changed.accepted_headers[0].hash = "d".repeat(64); + let other_network = Pass::new(changed, Some(cp.clone()), Limits::default()).unwrap(); + assert!(!other_network.driver.chain.borrow().contains_key(&50)); + let mut tampered = cp; + let mut bytes = STANDARD.decode(&tampered.continuation).unwrap(); + *bytes.last_mut().unwrap() ^= 1; + tampered.continuation = STANDARD.encode(bytes); + assert!(Pass::new(scope(), Some(tampered), Limits::default()).is_err()); + let oversized = ObservedChain { + anchor: scope().anchor, + genesis: None, + headers: (0..MAX_OBSERVED_POINTS as u64 + 1) + .map(|h| (h, "b".repeat(64))) + .collect(), + }; + let bytes = bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&oversized) + .unwrap(); + assert!(bincode::DefaultOptions::new() + .with_fixint_encoding() + .deserialize::(&bytes) + .is_err()); +} +#[test] +fn budget_finish_preserves_held_checkpoint_and_fences_late_io() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + pass.driver.chain.borrow_mut().insert(50, "b".repeat(64)); + let cp = snapshot(&pass); + pass.held = Some(cp.clone()); + pass.finish_budget().unwrap(); + assert_eq!( + pass.advance().unwrap(), + Step::Checkpoint { + checkpoint: cp.clone() + } + ); + assert!(pass.accept(1, Reply::Unavailable).is_err()); + assert!(pass.accept_chain(50, "b".repeat(64)).is_err()); + pass.acknowledge(&cp.checkpoint_id).unwrap(); + assert_eq!( + pass.advance().unwrap(), + Step::Stopped { + status: Status::Incomplete { + reason: "budget".into() + } + } + ); + pass.cancel(); + assert!(pass.finish_budget().is_err()); + assert_eq!( + pass.advance().unwrap(), + Step::Stopped { + status: Status::Cancelled + } + ); + let mut untouched = Pass::new(scope(), None, Limits::default()).unwrap(); + untouched.finish_budget().unwrap(); + assert!(matches!( + untouched.advance().unwrap(), + Step::Checkpoint { .. } + )); +} +#[test] +fn unchanged_cold_incomplete_stops_without_self_referential_replay() { + let pass = Pass::new(scope(), None, Limits::default()).unwrap(); + let cp = snapshot(&pass); + let mut reopened = Pass::new(scope(), Some(cp.clone()), Limits::default()).unwrap(); + let Step::Exchange { request_id, .. } = reopened.advance().unwrap() else { + panic!("init required"); + }; + reopened.accept(request_id, Reply::Unavailable).unwrap(); + assert!(matches!( + reopened.advance().unwrap(), + Step::Stopped { + status: Status::Incomplete { .. } + } + )); + // A complete replay keeps the exact original receipt link and rewind + // metadata; resetting the canonical scan may require applying it again. + let recreated = snapshot(&reopened); + assert_eq!(recreated, cp); +} +#[test] +fn obsolete_tips_and_recut_boundaries_do_not_accumulate() { + let pass = Pass::new(scope(), None, Limits::default()).unwrap(); + for tip in 101..10000 { + let mut needed = std::collections::BTreeSet::from([0, tip]); + let boundary = 1 + tip % 90; + needed.insert(boundary); + pass.driver.chain.borrow_mut().insert(tip, "b".repeat(64)); + pass.driver + .chain + .borrow_mut() + .insert(boundary, "c".repeat(64)); + pass.driver.retain_points(&needed); + assert_eq!(pass.driver.chain.borrow().len(), 3); + assert!(pass + .driver + .chain + .borrow() + .keys() + .all(|h| needed.contains(h))); + } +} +#[test] +fn mismatched_independent_reply_is_durable_canonical_evidence() { + let pass = Pass::new(scope(), None, Limits::default()).unwrap(); + let driver = pass.driver.clone(); + let expected = "b".repeat(64); + let mut f = Box::pin(driver.point(50, &expected)); + let mut cx = Context::from_waker(noop_waker_ref()); + assert!(f.as_mut().poll(&mut cx).is_pending()); + pass.driver.io.borrow_mut().chain_reply = Some(Anchor { + height: 50, + hash: "c".repeat(64), + }); + assert!(matches!(f.as_mut().poll(&mut cx), Poll::Ready(Ok(())))); + assert_eq!(pass.driver.chain.borrow().get(&50), Some(&"c".repeat(64))); + let restored = Pass::new(scope(), Some(snapshot(&pass)), Limits::default()).unwrap(); + assert_eq!( + restored.driver.chain.borrow().get(&50), + Some(&"c".repeat(64)) + ); +} + +#[test] +fn unchanged_complete_snapshot_reuses_exact_prior_metadata_and_rejects_conflict() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + let script = hex::decode(&scope().scripts[0].script).unwrap(); + pass.store + .inner + .borrow_mut() + .add_scripts(&[ScriptEntry { + script: script.clone(), + origin: ScriptOrigin::Derived, + required_from: 0, + }]) + .unwrap(); + pass.store + .inner + .borrow_mut() + .commit_shard(crate::store::ShardCommit { + start_height: 0, + end_height: 100, + sealed: true, + terminal_block_hash: "a".repeat(64), + revision_digest: "b".repeat(64), + covered_scripts: vec![script], + ..Default::default() + }) + .unwrap(); + *pass.store.rewind.borrow_mut() = Some(crate::store::Anchor { + height: 0, + hash: scope().accepted_headers[0].hash.clone(), + }); + pass.finished = Some(Status::Complete); + let mut cp = snapshot(&pass); + cp.previous_checkpoint_id = Some("c".repeat(64)); + let mut reopened = Pass::new(scope(), Some(cp.clone()), Limits::default()).unwrap(); + reopened.finished = Some(Status::Complete); + assert_eq!(snapshot(&reopened), cp); + let mut conflicting = cp; + conflicting.covered_through = 99; + let mut conflict = Pass::new(scope(), Some(conflicting), Limits::default()).unwrap(); + conflict.finished = Some(Status::Complete); + let bytes = durable( + &conflict.scope, + &conflict.store.inner.borrow(), + &conflict.driver.chain.borrow(), + conflict.store.policy_limit.get(), + ) + .unwrap(); + assert!(conflict.checkpoint(&bytes).is_err()); +} + +#[test] +fn supported_domain_includes_disjoint_maps_history_and_three_anchors() { + let old: std::collections::BTreeSet<_> = (1..=MAX_OLD_ENDPOINTS as u64).collect(); + let needed = needed_points( + 10_000..10_000 + 2 * MAX_MAP_SHARDS as u64, + &old, + 50_000, + Some(49_999), + ); + assert_eq!(needed.len(), MAX_OBSERVED_POINTS); + let pass = Pass::new(scope(), None, Limits::default()).unwrap(); + *pass.driver.chain.borrow_mut() = needed.iter().map(|h| (*h, "b".repeat(64))).collect(); + let encoded = bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&ObservedChain { + anchor: Anchor { + height: 50_000, + hash: "b".repeat(64), + }, + genesis: Some("b".repeat(64)), + headers: pass.driver.chain.borrow().clone(), + }) + .unwrap(); + let restored: ObservedChain = bincode::DefaultOptions::new() + .with_fixint_encoding() + .deserialize(&encoded) + .unwrap(); + assert_eq!(restored.headers.len(), MAX_OBSERVED_POINTS); + for tip in 50_001..50_101 { + let recut = needed_points( + 20_000..20_000 + 2 * MAX_MAP_SHARDS as u64, + &old, + tip, + Some(tip - 1), + ); + pass.driver.retain_points(&recut); + for h in &recut { + pass.driver.chain.borrow_mut().insert(*h, "b".repeat(64)); + } + assert_eq!(pass.driver.chain.borrow().len(), MAX_OBSERVED_POINTS); + } +} +#[test] +fn excessive_history_metadata_is_closed_without_checkpoint_or_evidence_loss() { + let mut requested = scope(); + requested.anchor.height = 10_000; + requested.accepted_headers[1] = requested.anchor.clone(); + let p = Pass::new(requested.clone(), None, Limits::default()).unwrap(); + let script = hex::decode(&requested.scripts[0].script).unwrap(); + p.store + .inner + .borrow_mut() + .add_scripts(&[ScriptEntry { + script: script.clone(), + origin: ScriptOrigin::Derived, + required_from: 0, + }]) + .unwrap(); + let mut supported = None; + for h in 1..=MAX_OLD_ENDPOINTS as u64 + 1 { + p.store + .inner + .borrow_mut() + .commit_shard(crate::store::ShardCommit { + start_height: h, + end_height: h, + sealed: true, + shard_id: h, + terminal_block_hash: "b".repeat(64), + revision_digest: "c".repeat(64), + covered_scripts: vec![script.clone()], + ..Default::default() + }) + .unwrap(); + if h == MAX_OLD_ENDPOINTS as u64 { + supported = Some(snapshot(&p)); + assert_eq!( + retained_endpoints(&p.store.inner.borrow()).unwrap().len(), + MAX_OLD_ENDPOINTS + ); + } + } + let mut generated = Pass::new(requested.clone(), supported, Limits::default()).unwrap(); + generated + .store + .inner + .borrow_mut() + .commit_shard(crate::store::ShardCommit { + start_height: 9000, + end_height: 9000, + sealed: true, + shard_id: 9000, + terminal_block_hash: "b".repeat(64), + revision_digest: "c".repeat(64), + covered_scripts: vec![script.clone()], + ..Default::default() + }) + .unwrap(); + generated.future = None; + generated.finished = Some(Status::Complete); + let Step::Checkpoint { checkpoint } = generated.advance().unwrap() else { + panic!("new accepted state requires a durable incomplete checkpoint"); + }; + assert!(!checkpoint.complete); + generated.acknowledge(&checkpoint.checkpoint_id).unwrap(); + assert!( + matches!(generated.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + let restored_generated = + Pass::new(requested.clone(), Some(checkpoint), Limits::default()).unwrap(); + assert_eq!( + encoded(&generated.store.inner.borrow()).unwrap(), + encoded(&restored_generated.store.inner.borrow()).unwrap() + ); + let cp = snapshot(&p); + let original_state = encoded(&p.store.inner.borrow()).unwrap(); + for _ in 0..3 { + let mut cold = Pass::new(requested.clone(), Some(cp.clone()), Limits::default()).unwrap(); + assert_eq!(encoded(&cold.store.inner.borrow()).unwrap(), original_state); + assert!( + matches!(cold.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + cold.finish_budget().unwrap(); + assert!( + matches!(cold.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + assert_eq!(encoded(&cold.store.inner.borrow()).unwrap(), original_state); + assert!(cold.driver.io.borrow().pending.is_none()); + } +} + +fn history_cap_seed(pass: &mut Pass, events: Vec) { + let script = hex::decode(&scope().scripts[0].script).unwrap(); + pass.store + .add_scripts(&[ScriptEntry { + script: script.clone(), + origin: ScriptOrigin::Derived, + required_from: 0, + }]) + .unwrap(); + pass.store + .commit_shard(crate::store::ShardCommit { + start_height: 0, + end_height: 100, + terminal_block_hash: "a".repeat(64), + revision_digest: "b".repeat(64), + events, + covered_scripts: vec![script.clone()], + pending_upsert: vec![crate::store::PendingPages { + id: None, + shard_id: 0, + revision_digest: "b".repeat(64), + script, + cursor: crate::store::RunCursor { + child: 0, + row: 0, + older_runs: None, + }, + carried: vec![], + fragments: vec![], + dummy_debt: Some((2, 3)), + attempts: 1, + target_anchor: Some(crate::store::Anchor { + height: 100, + hash: "a".repeat(64), + }), + }], + ..Default::default() + }) + .unwrap(); +} +fn history_cap_event(index: u32) -> crate::store::StoredEvent { + crate::store::StoredEvent { + script: hex::decode(&scope().scripts[0].script).unwrap(), + event: transparent_events::TransparentEvent::Receive(transparent_events::ReceiveEvent { + metadata: None, + height: 1, + txid: transparent_events::Txid([7; 32]), + transaction_index: 1, + output_index: index, + value: 1, + coinbase: false, + }), + shard_id: 0, + revision_digest: "b".repeat(64), + } +} +fn history_cap_ack_baseline(pass: &mut Pass) -> Checkpoint { + pass.future = None; + pass.finished = Some(Status::Incomplete { + reason: "budget".into(), + }); + let Step::Checkpoint { checkpoint } = pass.advance().unwrap() else { + panic!("durable baseline required") + }; + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + pass.finished = None; + pass.final_ack = false; + checkpoint +} +fn history_cap_assert_cold(checkpoint: Checkpoint, repeats: usize) { + for _ in 0..repeats { + let mut cold = Pass::new(scope(), Some(checkpoint.clone()), Limits::default()).unwrap(); + assert!( + matches!(cold.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + assert!( + cold.driver.io.borrow().pending.is_none(), + "terminal cold resume must not allocate IO" + ); + assert_eq!( + cold.store.inner.borrow().pending().unwrap()[0].dummy_debt, + Some((2, 3)) + ); + cold.finish_budget().unwrap(); + assert!( + matches!(cold.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + } + let mut changed_scope = scope(); + changed_scope.generation += 1; + changed_scope.scope_id = "e".repeat(64); + let mut changed = Pass::new(changed_scope, Some(checkpoint), Limits::default()).unwrap(); + assert!(matches!(changed.advance().unwrap(), Step::Stopped { + status: Status::Incomplete { reason } + } if reason == "history_metadata_limit")); + assert!(changed.driver.io.borrow().pending.is_none()); +} +#[test] +fn history_cap_event_boundary_preserves_projection_debt_and_cold_terminal() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + history_cap_seed(&mut pass, (0..65_536).map(history_cap_event).collect()); + pass.store + .commit_shard(crate::store::ShardCommit { + events: vec![history_cap_event(0), history_cap_event(0)], + ..Default::default() + }) + .unwrap(); + let baseline = history_cap_ack_baseline(&mut pass); + assert_eq!(baseline.records.len(), 65_536); + let mut retired = Pass::new(scope(), Some(baseline.clone()), Limits::default()).unwrap(); + retired + .store + .rollback_above( + &crate::store::Anchor { + height: 0, + hash: scope().accepted_headers[0].hash.clone(), + }, + "fixture", + ) + .unwrap(); + retired + .store + .commit_shard(crate::store::ShardCommit { + events: vec![history_cap_event(65_536)], + ..Default::default() + }) + .unwrap(); + assert_eq!(retired.store.events().unwrap().len(), 1); + assert!(!retired.store.policy_limit.get()); + let before = encoded(&pass.store.inner.borrow()).unwrap(); + let pending = pass.store.inner.borrow().pending().unwrap()[0].id.unwrap(); + let refused = pass.store.commit_shard(crate::store::ShardCommit { + events: vec![history_cap_event(65_536)], + pending_complete: vec![pending], + ..Default::default() + }); + let Step::Checkpoint { checkpoint } = pass.advance().expect( + "event policy limit must yield a durable terminal checkpoint, not event_limit exception", + ) else { + panic!("terminal checkpoint required") + }; + assert!(refused.is_err()); + assert!(!checkpoint.complete); + assert_eq!(checkpoint.records, baseline.records); + assert_eq!( + encoded(&pass.store.inner.borrow()).unwrap(), + before, + "refused commit cannot delete owed debt or accepted facts" + ); + assert_eq!( + checkpoint.previous_checkpoint_id, + Some(baseline.checkpoint_id) + ); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + assert!( + matches!(pass.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + history_cap_assert_cold(checkpoint, 3); +} +#[test] +fn history_cap_byte_boundary_preserves_journal_and_cold_terminal() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + history_cap_seed(&mut pass, vec![]); + let keys: Vec<_> = (0..8).map(|n| format!("{n:064x}")).collect(); + for key in &keys { + pass.store + .put_filter(key, &"c".repeat(64), false, &[]) + .unwrap(); + } + let overhead = encoded(&pass.store.inner.borrow()).unwrap().len(); + let mut remaining = 63 * 1024 * 1024 - overhead; + let mut last = 0; + for key in &keys { + let n = remaining.min(8 * 1024 * 1024); + remaining -= n; + last = n; + pass.store + .put_filter(key, &"c".repeat(64), false, &vec![0; n]) + .unwrap(); + } + assert_eq!(remaining, 0); + assert_eq!( + encoded(&pass.store.inner.borrow()).unwrap().len(), + 63 * 1024 * 1024 + ); + let baseline = history_cap_ack_baseline(&mut pass); + let refused = pass.store.put_filter( + keys.last().unwrap(), + &"c".repeat(64), + false, + &vec![0; last + 1], + ); + let Step::Checkpoint { checkpoint } = pass + .advance() + .expect("metadata policy must yield terminal checkpoint") + else { + panic!("terminal checkpoint required") + }; + assert!(refused.is_err()); + assert_eq!(checkpoint.records, baseline.records); + assert!(!checkpoint.complete); + assert_eq!( + encoded(&pass.store.inner.borrow()).unwrap().len(), + 63 * 1024 * 1024 + ); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + assert!( + matches!(pass.advance().unwrap(),Step::Stopped{status:Status::Incomplete{reason}} if reason=="history_metadata_limit") + ); + history_cap_assert_cold(checkpoint, 1); + pass.store + .inner + .borrow_mut() + .put_filter( + keys.last().unwrap(), + &"c".repeat(64), + false, + &vec![0; last + 1], + ) + .unwrap(); + let legacy = history_cap_v3_checkpoint(&pass); + assert!(STANDARD.decode(&legacy.continuation).unwrap().len() < MAX_CONTINUATION_BYTES); + let before = encoded(&pass.store.inner.borrow()).unwrap(); + history_cap_assert_cold(legacy.clone(), 1); + let cold = Pass::new(scope(), Some(legacy), Limits::default()).unwrap(); + assert_eq!(encoded(&cold.store.inner.borrow()).unwrap(), before); + // The real failed-walk path changes only its existing pending debt and one + // failure count. Start from the acknowledged, nonterminal at-cap baseline. + pass = Pass::new(scope(), Some(baseline.clone()), Limits::default()).unwrap(); + assert!(!pass.store.policy_limit.get()); + let mut owed = pass.store.pending().unwrap().remove(0); + owed.dummy_debt = Some((4, 5)); + let before = encoded(&pass.store.inner.borrow()).unwrap(); + let mut invalid = owed.clone(); + invalid.cursor.row += 1; + assert!(matches!( + pass.store.commit_shard(crate::store::ShardCommit { + revision_digest: owed.revision_digest.clone(), + failed_entries: vec![owed.script.clone()], + pending_upsert: vec![invalid], + ..Default::default() + }), + Err(crate::store::StoreError::HistoryMetadataLimit) + )); + assert_eq!(encoded(&pass.store.inner.borrow()).unwrap(), before); + pass = Pass::new(scope(), Some(baseline.clone()), Limits::default()).unwrap(); + assert!(!pass.store.policy_limit.get()); + pass.store + .commit_shard(crate::store::ShardCommit { + revision_digest: owed.revision_digest.clone(), + failed_entries: vec![owed.script.clone()], + pending_upsert: vec![owed], + ..Default::default() + }) + .unwrap(); + assert!(pass.store.policy_limit.get()); + let Step::Checkpoint { checkpoint } = pass.advance().unwrap() else { + panic!("newly owed cover work requires a terminal durable checkpoint"); + }; + assert!(!checkpoint.complete); + assert_eq!(checkpoint.records, baseline.records); + assert!(pass.driver.io.borrow().pending.is_none()); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + let mut cold = Pass::new(scope(), Some(checkpoint), Limits::default()).unwrap(); + assert!(matches!(cold.advance().unwrap(), Step::Stopped { + status: Status::Incomplete {reason} + } if reason == "history_metadata_limit")); + assert_eq!(cold.store.pending().unwrap()[0].dummy_debt, Some((4, 5))); + assert!(cold.driver.io.borrow().pending.is_none()); +} + +fn history_cap_v3_checkpoint(pass: &Pass) -> Checkpoint { + let mut cp = snapshot(pass); + let bytes = bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&LegacyContinuationV3 { + version: 3, + source_id: pass.scope.source_id.clone(), + state: pass.store.inner.borrow().clone(), + chain: ObservedChain { + anchor: pass.scope.anchor.clone(), + genesis: pass.driver.chain.borrow().get(&0).cloned(), + headers: pass.driver.chain.borrow().clone(), + }, + }) + .unwrap(); + cp.checkpoint_id = checkpoint_id( + &cp.source_id, + &cp.scope_id, + cp.generation, + &cp.anchor, + cp.complete, + &bytes, + ); + cp.continuation = STANDARD.encode(bytes); + cp +} + +#[test] +fn history_cap_v3_migration_preserves_exact_state_debt_and_source_digest() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + history_cap_seed(&mut pass, vec![history_cap_event(0)]); + pass.driver.chain.borrow_mut().insert(50, "b".repeat(64)); + let cp = history_cap_v3_checkpoint(&pass); + let before = encoded(&pass.store.inner.borrow()).unwrap(); + let restored = Pass::new(scope(), Some(cp.clone()), Limits::default()).unwrap(); + assert_eq!(encoded(&restored.store.inner.borrow()).unwrap(), before); + assert_eq!( + restored.driver.chain.borrow().get(&50), + Some(&"b".repeat(64)) + ); + assert!(!restored.store.policy_limit.get()); + let upgraded = snapshot(&restored); + assert_eq!( + &STANDARD.decode(&upgraded.continuation).unwrap()[..4], + &4u32.to_le_bytes() + ); + let cold = Pass::new(scope(), Some(upgraded), Limits::default()).unwrap(); + assert_eq!(encoded(&cold.store.inner.borrow()).unwrap(), before); + let mut tampered = cp.clone(); + tampered.checkpoint_id = "f".repeat(64); + assert!(Pass::new(scope(), Some(tampered), Limits::default()).is_err()); + let mut other_source = scope(); + other_source.source_id = "e".repeat(64); + assert!(Pass::new(other_source, Some(cp), Limits::default()).is_err()); +} + +#[test] +fn history_cap_reserve_encodes_the_maximum_supported_header_domain() { + let mut maximal = scope(); + maximal.anchor.height = u64::MAX; + let headers: BTreeMap<_, _> = (0..MAX_OBSERVED_POINTS as u64 - 1) + .chain(std::iter::once(u64::MAX)) + .map(|height| (height, "f".repeat(64))) + .collect(); + assert_eq!(headers.len(), MAX_OBSERVED_POINTS); + let memory = MemoryStore::new(); + let continuation = durable(&maximal, &memory, &headers, true).unwrap(); + // Measure the actual fixed-int serializer, including both anchors, source, + // genesis, version and terminal flag. This reserve is independent of state. + let overhead = continuation.len() - encoded(&memory).unwrap().len(); + assert!(overhead < 512 * 1024); + assert!(crate::shared_store::MAX_TERMINAL_STORE_BYTES + overhead < MAX_CONTINUATION_BYTES); +} + +#[test] +fn history_cap_invalid_commit_is_not_a_local_policy_failure_and_cancel_wins() { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + history_cap_seed(&mut pass, vec![history_cap_event(0)]); + pass.store.inner.borrow_mut().set_event_limit(1); + let before = encoded(&pass.store.inner.borrow()).unwrap(); + let mut invalid = history_cap_event(0); + if let transparent_events::TransparentEvent::Receive(ref mut event) = invalid.event { + event.value = 2; + } + let result = pass.store.commit_shard(crate::store::ShardCommit { + events: vec![history_cap_event(1), invalid], + ..Default::default() + }); + assert!(result.is_err()); + assert!(!matches!( + result, + Err(crate::store::StoreError::HistoryMetadataLimit) + )); + assert!(!pass.store.policy_limit.get()); + assert_eq!(encoded(&pass.store.inner.borrow()).unwrap(), before); + let refused = pass.store.commit_shard(crate::store::ShardCommit { + events: vec![history_cap_event(1)], + ..Default::default() + }); + assert!(matches!( + refused, + Err(crate::store::StoreError::HistoryMetadataLimit) + )); + pass.cancel(); + assert!(matches!( + pass.advance().unwrap(), + Step::Stopped { + status: Status::Cancelled + } + )); + assert!(pass.finish_budget().is_err()); + assert_eq!(encoded(&pass.store.inner.borrow()).unwrap(), before); +} + +#[test] +fn history_cap_cover_failure_encoder_reserve_bounds_all_pending_work() { + let mut state = MemoryStore::new(); + let pending: Vec<_> = (0..crate::memory_store::DEFAULT_PENDING_LIMIT) + .map(|index| { + let mut script = vec![0; 40]; + script[..8].copy_from_slice(&(index as u64).to_le_bytes()); + crate::store::PendingPages { + id: None, + shard_id: 0, + revision_digest: "b".repeat(64), + script, + cursor: crate::store::RunCursor { + child: 0, + row: 0, + older_runs: None, + }, + carried: vec![], + fragments: vec![], + dummy_debt: None, + attempts: 0, + target_anchor: Some(crate::store::Anchor { + height: 100, + hash: "a".repeat(64), + }), + } + }) + .collect(); + state + .commit_shard(crate::store::ShardCommit { + pending_upsert: pending, + ..Default::default() + }) + .unwrap(); + let before = encoded(&state).unwrap().len(); + let mut pending = state.pending().unwrap(); + for item in &mut pending { + item.dummy_debt = Some((u8::MAX, u32::MAX)); + } + let scripts = pending.iter().map(|item| item.script.clone()).collect(); + state + .commit_shard(crate::store::ShardCommit { + failed_entries: scripts, + pending_upsert: pending, + ..Default::default() + }) + .unwrap(); + let delta = encoded(&state).unwrap().len() - before; + assert_eq!(delta, crate::memory_store::DEFAULT_PENDING_LIMIT * 65); + assert!(delta < crate::shared_store::MAX_TERMINAL_STORE_BYTES - MAX_STORE_BYTES); +} + +#[test] +fn history_cap_transport_and_malformed_reply_are_not_terminal_policy() { + for reply in [ + Reply::Unavailable, + Reply::Data { + body: "invalid-base64!".into(), + }, + ] { + let mut pass = Pass::new(scope(), None, Limits::default()).unwrap(); + let Step::Exchange { request_id, .. } = pass.advance().unwrap() else { + panic!("init"); + }; + pass.accept(request_id, reply).unwrap(); + let Step::Checkpoint { checkpoint } = pass.advance().unwrap() else { + panic!("failure checkpoint"); + }; + assert!(!pass.store.policy_limit.get()); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + assert!(matches!(pass.advance().unwrap(), Step::Stopped { + status: Status::Incomplete {reason} + } if reason != "history_metadata_limit")); + } +} diff --git a/crates/z-transparent-pir-v2/src/serialization.rs b/crates/z-transparent-pir-v2/src/serialization.rs new file mode 100644 index 0000000..320a7f5 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/serialization.rs @@ -0,0 +1,89 @@ +use crate::store::{receive_key, spend_key, StoredEvent}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeMap; +use transparent_events::TransparentEvent; +pub(crate) mod event { + use super::*; + pub fn serialize(v: &TransparentEvent, s: S) -> Result { + v.to_bytes().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + let b = Vec::::deserialize(d)?; + TransparentEvent::from_bytes(&b).map_err(serde::de::Error::custom) + } +} +pub(crate) mod events { + use super::*; + pub fn serialize(v: &[TransparentEvent], s: S) -> Result { + v.iter() + .map(|e| e.to_bytes()) + .collect::>() + .serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + Vec::>::deserialize(d)? + .iter() + .map(|b| TransparentEvent::from_bytes(b).map_err(serde::de::Error::custom)) + .collect() + } +} +pub(crate) mod table { + use super::*; + use crate::client::Table; + pub fn serialize(v: &Table, s: S) -> Result { + match v { + Table::Directory => None, + Table::Pages(i) => Some(*i), + } + .serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + Ok(match Option::::deserialize(d)? { + None => Table::Directory, + Some(i) if i < 32 => Table::Pages(i), + _ => return Err(serde::de::Error::custom("invalid table")), + }) + } +} +pub(crate) mod receives { + use super::*; + use transparent_events::Txid; + pub fn serialize( + v: &BTreeMap<(Txid, u32), StoredEvent>, + s: S, + ) -> Result { + v.values().collect::>().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + Vec::::deserialize(d)? + .into_iter() + .map(|e| match e.event { + TransparentEvent::Receive(r) => Ok((receive_key(&r), e)), + _ => Err(serde::de::Error::custom("invalid receive")), + }) + .collect() + } +} +pub(crate) mod spends { + use super::*; + use transparent_events::Txid; + pub fn serialize( + v: &BTreeMap<(Txid, u32, Txid, u32), StoredEvent>, + s: S, + ) -> Result { + v.values().collect::>().serialize(s) + } + pub fn deserialize<'de, D: Deserializer<'de>>( + d: D, + ) -> Result, D::Error> { + Vec::::deserialize(d)? + .into_iter() + .map(|e| match e.event { + TransparentEvent::Spend(r) => Ok((spend_key(&r), e)), + _ => Err(serde::de::Error::custom("invalid spend")), + }) + .collect() + } +} diff --git a/crates/z-transparent-pir-v2/src/shared_store.rs b/crates/z-transparent-pir-v2/src/shared_store.rs new file mode 100644 index 0000000..2c6df57 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/shared_store.rs @@ -0,0 +1,184 @@ +use crate::memory_store::MemoryStore; +use crate::store::*; +use bincode::Options; +use std::{ + cell::{Cell, RefCell}, + rc::Rc, +}; +pub(crate) const MAX_RETAINED_EVENTS: usize = 65_536; +pub(crate) const MAX_STORE_BYTES: usize = 63 * 1024 * 1024; +pub(crate) const MAX_TERMINAL_STORE_BYTES: usize = MAX_STORE_BYTES + 512 * 1024; +pub(crate) const MAX_CONTINUATION_BYTES: usize = 64 * 1024 * 1024; +#[derive(Clone)] +pub(crate) struct SharedStore { + pub inner: Rc>, + pub rewind: Rc>>, + pub policy_limit: Rc>, +} +impl SharedStore { + // A refused growth mutation leaves the complete journal, including owed + // cover work, unchanged. Only typed local caps set the terminal flag. + fn update( + &mut self, + f: impl FnOnce(&mut MemoryStore) -> Result, + ) -> Result { + self.update_with_cover_reserve(false, f) + } + fn update_with_cover_reserve( + &mut self, + cover_failure: bool, + f: impl FnOnce(&mut MemoryStore) -> Result, + ) -> Result { + let mut candidate = self.inner.borrow().clone(); + let result = f(&mut candidate).and_then(|value| { + let bytes = bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialized_size(&candidate) + .map_err(|_| StoreError::Corrupt("store encoding".into()))?; + if bytes > MAX_STORE_BYTES as u64 { + if !cover_failure || bytes > MAX_TERMINAL_STORE_BYTES as u64 { + return Err(StoreError::HistoryMetadataLimit); + } + // The failed reply has already created this obligation. Preserve + // it in reserved space, then stop before returning another IO. + self.policy_limit.set(true); + } + *self.inner.borrow_mut() = candidate; + Ok(value) + }); + if matches!(result, Err(StoreError::HistoryMetadataLimit)) { + self.policy_limit.set(true); + } + result + } + fn cover_failure(&self, commit: &ShardCommit) -> Result { + let [pending] = commit.pending_upsert.as_slice() else { + return Ok(false); + }; + if commit.failed_entries.as_slice() != [pending.script.clone()] + || pending.script.len() > 40 + || pending.dummy_debt.is_none() + || !commit.events.is_empty() + || !commit.covered_scripts.is_empty() + || !commit.pending_complete.is_empty() + || !commit.cached_runs.is_empty() + || pending.shard_id != commit.shard_id + || pending.revision_digest != commit.revision_digest + { + return Ok(false); + } + let Some(mut prior) = self + .inner + .borrow() + .pending()? + .into_iter() + .find(|held| held.id.is_some() && held.id == pending.id) + else { + return Ok(false); + }; + prior.dummy_debt = pending.dummy_debt; + Ok(prior == *pending) + } +} +impl WalletStore for SharedStore { + fn set_identity(&self) -> Result, StoreError> { + self.inner.borrow().set_identity() + } + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + self.update(|state| state.bind_set(identity)) + } + fn anchor(&self) -> Result, StoreError> { + self.inner.borrow().anchor() + } + fn scripts(&self) -> Result, StoreError> { + self.inner.borrow().scripts() + } + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + self.update(|state| state.add_scripts(entries)) + } + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + self.inner.borrow().coverage(script) + } + fn provisional(&self) -> Result, StoreError> { + self.inner.borrow().provisional() + } + fn events(&self) -> Result, StoreError> { + self.inner.borrow().events() + } + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + let cover_failure = self.cover_failure(&commit)?; + self.update_with_cover_reserve(cover_failure, |state| state.commit_shard(commit)) + } + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + self.update(|state| state.commit_anchor(anchor, settled_through, covered_through)) + } + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result { + let result = self.update(|state| state.rollback_above(anchor, reason))?; + let mut rewind = self.rewind.borrow_mut(); + if rewind + .as_ref() + .is_none_or(|held| anchor.height < held.height) + { + *rewind = Some(anchor.clone()); + } + Ok(result) + } + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + self.inner + .borrow_mut() + .promote_provisional(shard_id, revision_digest) + } + fn pending(&self) -> Result, StoreError> { + self.inner.borrow().pending() + } + fn pending_limit(&self) -> usize { + self.inner.borrow().pending_limit() + } + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + self.inner.borrow().cached_run(child_digest, script) + } + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + self.inner.borrow().setup(key) + } + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + self.update(|state| state.put_setup(key, blob)) + } + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + self.inner.borrow_mut().drop_setup(key) + } + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + self.inner.borrow().filter(revision_digest, filter_hash) + } + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + self.update(|state| state.put_filter(revision_digest, filter_hash, sealed, bytes)) + } + fn last_commit(&self) -> Result { + self.inner.borrow().last_commit() + } + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + self.inner.borrow().entry_failures(shard_id, script) + } +} diff --git a/crates/z-transparent-pir-v2/src/store.rs b/crates/z-transparent-pir-v2/src/store.rs new file mode 100644 index 0000000..d9ef121 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/store.rs @@ -0,0 +1,561 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! What a wallet keeps between syncs, and the one boundary it changes at. +//! +//! A sync that starts from nothing every time cannot be a returning wallet: it +//! forgets the receive it found last month, so the spend it finds today is +//! unresolved, and moving the birthday forward to save work throws the receive +//! away for good. The store is the wallet's memory of what it has recovered, +//! which scripts it is responsible for and over which ranges each is covered, +//! what it accepted as the chain's anchor, and what private work it still owes. +//! +//! # One commit per shard +//! +//! Everything a shard yields is committed together — its events, the coverage +//! it establishes for each script, and the page work still owed — or not at +//! all. A crash between two commits loses at most one shard's retrieval, which +//! the next sync repeats; it never records coverage for events it did not keep +//! or events under coverage it did not record. Retrying a commit is idempotent +//! by construction: events are keyed by their stable identities, coverage by +//! script and range, and a repeat that *differs* in any field is a +//! contradiction, refused rather than merged. +//! +//! # Provisional and settled +//! +//! Coverage taken from an unsealed tail carries that revision's digest. When +//! the tail is superseded, that coverage and the events under it are truncated +//! and the range re-derived; when the same digest is later sealed, it is +//! promoted in place. Settled coverage is never rewritten except by a reorg, +//! which rolls everything above the accepted ancestor back at once. +//! +//! # What the wallet owns +//! +//! This is a trait so the wallet that embeds the library supplies the storage: +//! its own database, its own transaction, its own encryption. The reference +//! implementations — in memory here, SQLite in `transparent-wallet-store` — +//! are what the tests run against and what a first integration can use as-is. + +use crate::client::Table; +use crate::ledger::{Ledger, LedgerError}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use transparent_events::TransparentEvent; +use transparent_filter::{SealParameters, ShardMap}; + +/// The publication lineage a store is bound to. +/// +/// Everything in the map that does not change as the set grows: chain, +/// profile, envelope version, first height, and the seal thresholds each +/// geometry was sealed under. The map digest itself changes with every tail +/// republication and is deliberately not part of this. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetIdentity { + pub shard_schema: String, + pub network: String, + pub genesis_hash: String, + pub profile: String, + pub range_envelope_version: u16, + pub start_height: u64, + pub seal: BTreeMap, +} + +impl SetIdentity { + pub fn of_schema(map: &ShardMap, schema: &str) -> Self { + Self { + shard_schema: schema.to_string(), + network: map.network.clone(), + genesis_hash: map.genesis_hash.clone(), + profile: map.profile.clone(), + range_envelope_version: map.range_envelope_version, + start_height: map.start_height, + seal: map.seal.clone(), + } + } + + /// SHA-256 of the canonical serialization. + pub fn digest(&self) -> String { + hex::encode(Sha256::digest( + serde_json::to_vec(self).expect("a set identity serializes"), + )) + } + + /// Whether `other` is the same lineage, allowing a geometry the store had + /// not seen to appear (a set growing into another tier) but never a + /// changed seal for one already in use. + pub fn continues(&self, other: &Self) -> bool { + self.shard_schema == other.shard_schema + && self.network == other.network + && self.genesis_hash == other.genesis_hash + && self.profile == other.profile + && self.range_envelope_version == other.range_envelope_version + && self.start_height == other.start_height + && self + .seal + .iter() + .all(|(name, seal)| other.seal.get(name).is_none_or(|theirs| theirs == seal)) + } +} + +/// The chain block a wallet has accepted as the end of its coverage. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Anchor { + pub height: u64, + /// Display hex. + pub hash: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ScriptOrigin { + /// From the wallet's own derivation rules. + Derived, + /// Imported by the user; its history may begin anywhere. + Imported, +} + +/// One script the wallet is responsible for, and the height its required +/// coverage begins at. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScriptEntry { + pub script: Vec, + pub origin: ScriptOrigin, + /// First height the wallet needs covered for this script. The wallet's + /// own rule decides it; a derived script's is typically the birthday, + /// an imported script's is the set's first height. + pub required_from: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CoverageKind { + /// From a sealed shard. Immutable short of a reorg. + Settled, + /// From an unsealed tail revision. Replaced when the tail is republished. + Provisional, +} + +/// One script's coverage over one shard's range. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CoverageRange { + pub script: Vec, + pub start_height: u64, + pub end_height: u64, + pub kind: CoverageKind, + pub shard_id: u64, + pub revision_digest: String, + pub terminal_block_hash: String, + /// Full published endpoint, distinct from the covered endpoint. + pub source_anchor: Option, +} + +/// One event the wallet keeps, with where it came from. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoredEvent { + pub script: Vec, + #[serde(with = "crate::serialization::event")] + pub event: TransparentEvent, + pub shard_id: u64, + pub revision_digest: String, +} + +/// Last committed fragment's boundary, needed to validate greedy packing after resume. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PageBoundary { + pub event_bytes: u32, + #[serde(with = "crate::serialization::event")] + pub last_event: TransparentEvent, +} + +/// Where a chain walk resumes: the run it reads next. +/// +/// A script's history in a directory shard is a chain of runs, newest first, +/// each in an earlier page shard than the one before. The directory entry +/// names the newest; every run's first fragment names the next older one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RunCursor { + /// The page shard `j` holding the run; its table is `pages-`. + pub child: u8, + /// Zero-based row of the run's first fragment in that table. + pub row: u32, + /// How many runs the run's first fragment must report as older than it: + /// one fewer than the newer run reported. `None` for the newest run, + /// which the directory entry names without a count. + pub older_runs: Option, +} + +/// Page retrieval a wallet still owes for one script in one directory shard +/// revision: the rest of its chain of runs. +/// +/// Every run newer than the cursor has been read, checked and committed. Kept +/// durably so an exhausted budget or an outage leaves resumable work, not a +/// synchronized balance. Keyed by the directory shard revision alone, whose +/// digest commits to every page shard. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PendingPages { + /// Assigned by the store; `None` until first committed. + pub id: Option, + pub shard_id: u64, + pub revision_digest: String, + pub script: Vec, + /// The run the walk reads next. + pub cursor: RunCursor, + /// The two oldest events already read, oldest first: the newer runs' or, + /// before any run was read, the directory entry's inline events. The run + /// at the cursor must be older than both, and each of its events was + /// pushed out by the event two places after it, which these supply for + /// its last two. + #[serde(with = "crate::serialization::events")] + pub carried: Vec, + /// Checked fragments of the current run, each encoded as a page row holding + /// only this script. They are revalidated on resume and cleared at the next run. + pub fragments: Vec>, + /// Logical cover traffic owed after malformed content; never a PIR secret or query. + /// Bound to this pending script, revision and accepted target. + pub dummy_debt: Option<(u8, u32)>, + pub attempts: u32, + pub target_anchor: Option, +} + +/// One script's run in a sealed page shard, as a wallet verified it, and the +/// name of the cached run below it. +/// +/// Keyed by the page shard's chained identity +/// ([`ShardManifest::child_digests`](transparent_shard::ShardManifest::child_digests)), +/// which covers that page shard and every older one in its directory shard, so +/// nothing the run or the runs below it depend on can change under the key. +/// Following `older` back to the start of the directory shard gives the +/// script's history there, and a walk that reaches a cached run stops at it. +/// +/// A store drops a directory shard's cached runs for a script once it covers +/// that script over the shard sealed: no later walk of the shard is expected. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CachedRun { + /// The page shard's chained identity, in hex. + pub child_digest: String, + pub script: Vec, + /// The directory shard the run belongs to. + pub shard_id: u64, + /// The run's own events, oldest first. + #[serde(with = "crate::serialization::events")] + pub events: Vec, + /// The chained identity of the page shard holding the script's previous + /// run, or `None` when this is its oldest run in the directory shard. + pub older: Option, +} + +/// What one shard's retrieval commits, all at once. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ShardCommit { + /// Original publication endpoint when coverage is clipped to a wallet anchor. + pub source_anchor: Option, + pub shard_id: u64, + pub revision_digest: String, + pub sealed: bool, + pub start_height: u64, + pub end_height: u64, + pub terminal_block_hash: String, + pub events: Vec, + /// Scripts whose coverage this commit extends over the shard's range. + pub covered_scripts: Vec>, + /// Pending page work created or advanced by this commit. + pub pending_upsert: Vec, + /// Pending ids this commit finishes. + pub pending_complete: Vec, + /// Verified runs to keep for later walks. + pub cached_runs: Vec, + /// Scripts whose directory entry in this shard failed on content in this + /// commit's retrieval: a reply that did not decode or verify, or a record + /// that contradicts what the store holds. Each adds one to the entry's + /// [`WalletStore::entry_failures`]; unlike everything else here, that is + /// not idempotent, so a commit carrying failures is made once. + pub failed_entries: Vec>, +} + +impl ShardCommit { + /// Whether this commit covers its `covered_scripts` over the whole of a + /// sealed shard, so no later walk of the shard is expected for them and + /// the store drops their cached runs in it. Coverage clipped to a wallet + /// anchor inside the shard does not: the next sync carries on through it. + pub fn finishes_shard(&self) -> bool { + self.sealed + && self + .source_anchor + .as_ref() + .is_none_or(|source| source.height == self.end_height) + } +} + +/// Published setup for one table segment, kept for reuse. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SetupBlob { + pub public_params_base64: String, + pub public_params_sha256: String, +} + +/// What a setup is keyed by: the set lineage, the exact revision, the table +/// and the segment. Anything less would reuse parameters against bytes they +/// were not derived from. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SetupKey { + pub set_digest: String, + pub revision_digest: String, + #[serde(with = "crate::serialization::table")] + pub table: Table, + pub segment: u32, +} + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + /// Local supported-history policy; no part of the refused mutation is applied. + #[error("local history metadata limit")] + HistoryMetadataLimit, + #[error("store is corrupt: {0}")] + Corrupt(String), + /// A repeat that differs. Nothing is written. + #[error("contradiction: {0}")] + Contradiction(#[from] LedgerError), + #[error("the store is bound to set {stored}, this map is {offered}")] + SetMismatch { stored: String, offered: String }, + #[error("the commit would leave more than {limit} pending page retrievals")] + PendingLimit { limit: usize }, +} + +/// The wallet's durable memory. See the module documentation. +pub trait WalletStore { + /// The lineage this store is bound to, with its digest, if any. + fn set_identity(&self) -> Result, StoreError>; + /// Binds a fresh store to a lineage, or confirms a bound one continues it. + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError>; + + fn anchor(&self) -> Result, StoreError>; + + fn scripts(&self) -> Result, StoreError>; + /// Adds scripts. An existing script's `required_from` is never raised: + /// coverage once required stays required. + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result; + + fn coverage(&self, script: &[u8]) -> Result, StoreError>; + /// Every provisional range across every script. + fn provisional(&self) -> Result, StoreError>; + + /// Every kept event in canonical order. + fn events(&self) -> Result, StoreError>; + /// The ledger replayed from `events`. + fn ledger(&self) -> Result { + let mut events: Vec<(Vec, TransparentEvent)> = self + .events()? + .into_iter() + .map(|stored| (stored.script, stored.event)) + .collect(); + let mut ledger = Ledger::new(); + ledger.replay(&mut events)?; + Ok(ledger) + } + + /// The one mutating boundary for retrieval. Atomic and idempotent; a + /// repeat that differs is refused whole. + fn commit_shard(&mut self, commit: ShardCommit) -> Result; + /// Records the accepted anchor and derived coverage summaries. + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result; + /// Removes every event, coverage range and pending item above `height` + /// and lowers the anchor to it. A reorg or a replaced provisional tail. + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result; + /// Provisional coverage under `revision_digest` becomes settled: the same + /// bytes were sealed. + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError>; + + fn pending(&self) -> Result, StoreError>; + /// The run cached for `script` in the page shard with chained identity + /// `child_digest`, if any. See [`CachedRun`]. + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError>; + /// The most pending page retrievals a commit may leave behind. + fn pending_limit(&self) -> usize; + + fn setup(&self, key: &SetupKey) -> Result, StoreError>; + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError>; + /// Forgets a setup a reply showed to be stale, so the next sync fetches + /// the published one again. Forgetting one not held is not an error. + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError>; + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError>; + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError>; + + /// The id of the last commit, zero before any. + fn last_commit(&self) -> Result; + + /// Content failures recorded for `script`'s directory entry in shard + /// `shard_id`, across every sync: the number of commits that named it in + /// [`ShardCommit::failed_entries`]. Never reset by a later success, a + /// new revision of the shard or a rollback, so the cap on attempts holds + /// whatever the service does in between. + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result; +} + +impl WalletStore for Box { + fn set_identity(&self) -> Result, StoreError> { + (**self).set_identity() + } + fn bind_set(&mut self, identity: &SetIdentity) -> Result<(), StoreError> { + (**self).bind_set(identity) + } + fn anchor(&self) -> Result, StoreError> { + (**self).anchor() + } + fn scripts(&self) -> Result, StoreError> { + (**self).scripts() + } + fn add_scripts(&mut self, entries: &[ScriptEntry]) -> Result { + (**self).add_scripts(entries) + } + fn coverage(&self, script: &[u8]) -> Result, StoreError> { + (**self).coverage(script) + } + fn provisional(&self) -> Result, StoreError> { + (**self).provisional() + } + fn events(&self) -> Result, StoreError> { + (**self).events() + } + fn commit_shard(&mut self, commit: ShardCommit) -> Result { + (**self).commit_shard(commit) + } + fn commit_anchor( + &mut self, + anchor: &Anchor, + settled_through: u64, + covered_through: u64, + ) -> Result { + (**self).commit_anchor(anchor, settled_through, covered_through) + } + fn rollback_above(&mut self, anchor: &Anchor, reason: &str) -> Result { + (**self).rollback_above(anchor, reason) + } + fn promote_provisional( + &mut self, + shard_id: u64, + revision_digest: &str, + ) -> Result<(), StoreError> { + (**self).promote_provisional(shard_id, revision_digest) + } + fn pending(&self) -> Result, StoreError> { + (**self).pending() + } + fn pending_limit(&self) -> usize { + (**self).pending_limit() + } + fn cached_run( + &self, + child_digest: &str, + script: &[u8], + ) -> Result, StoreError> { + (**self).cached_run(child_digest, script) + } + fn setup(&self, key: &SetupKey) -> Result, StoreError> { + (**self).setup(key) + } + fn put_setup(&mut self, key: &SetupKey, blob: &SetupBlob) -> Result<(), StoreError> { + (**self).put_setup(key, blob) + } + fn drop_setup(&mut self, key: &SetupKey) -> Result<(), StoreError> { + (**self).drop_setup(key) + } + fn filter( + &self, + revision_digest: &str, + filter_hash: &str, + ) -> Result>, StoreError> { + (**self).filter(revision_digest, filter_hash) + } + fn put_filter( + &mut self, + revision_digest: &str, + filter_hash: &str, + sealed: bool, + bytes: &[u8], + ) -> Result<(), StoreError> { + (**self).put_filter(revision_digest, filter_hash, sealed, bytes) + } + fn last_commit(&self) -> Result { + (**self).last_commit() + } + fn entry_failures(&self, shard_id: u64, script: &[u8]) -> Result { + (**self).entry_failures(shard_id, script) + } +} + +/// The identity a receive is keyed by. +pub fn receive_key(event: &transparent_events::ReceiveEvent) -> (transparent_events::Txid, u32) { + (event.txid, event.output_index) +} + +/// The identity a spend is keyed by. +pub fn spend_key( + event: &transparent_events::SpendEvent, +) -> (transparent_events::Txid, u32, transparent_events::Txid, u32) { + ( + event.spending_txid, + event.input_index, + event.spent_txid, + event.spent_output_index, + ) +} + +/// Sorts one script's ranges and drops exact repeats. +/// +/// Ranges are kept per shard rather than merged into runs, deliberately: each +/// carries the block hash its coverage rests on, and a reorg is found by +/// asking the wallet's chain about those hashes newest first. Merging shards +/// 0-2 into one range would keep only shard 2's hash, and a reorg inside +/// shard 1 would then roll back to the set's start instead of to shard 0. +pub fn merge_coverage(mut ranges: Vec) -> Vec { + ranges.sort_by_key(|range| (range.start_height, range.end_height)); + ranges.dedup(); + ranges +} + +/// The heights in `[from, through]` that `ranges` do not cover, as inclusive +/// gaps in ascending order. +pub fn uncovered(ranges: &[CoverageRange], from: u64, through: u64) -> Vec<(u64, u64)> { + let mut sorted: Vec<&CoverageRange> = ranges.iter().collect(); + sorted.sort_by_key(|range| range.start_height); + let mut gaps = Vec::new(); + let mut next = from; + for range in sorted { + if range.end_height < next { + continue; + } + if range.start_height > next { + gaps.push((next, range.start_height.min(through + 1) - 1)); + } + next = next.max(range.end_height + 1); + if next > through { + break; + } + } + if next <= through { + gaps.push((next, through)); + } + gaps.retain(|(a, b)| a <= b); + gaps +} diff --git a/crates/z-transparent-pir-v2/src/sync.rs b/crates/z-transparent-pir-v2/src/sync.rs new file mode 100644 index 0000000..2242017 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/sync.rs @@ -0,0 +1,3945 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Birthday to balance: the join between public filters and private retrieval. +//! +//! This is the path the whole design exists to make cheap: +//! +//! 1. fetch the published shard map and binary-search it for the birthday; +//! 2. download **every** filter from that shard forward and match locally; +//! 3. for each matched script and directory shard, privately retrieve its +//! directory row, then walk the chain of runs its entry points to, newest +//! first, through the shard's page shards (see `walk_chains`); +//! 4. replay the events into a UTXO set. +//! +//! Step 2 downloads every filter in range, not only the ones that will match. +//! Which filters a wallet asks for must not be a function of its scripts. +//! +//! Step 3 addresses a row over the shard's whole logical row space, which is +//! its segments concatenated. Every segment answers, and the row is identified +//! by its exact script bytes rather than by which segment returned it: asking +//! only the segment that holds the script would disclose part of the script. +//! Coverage advances for a shard only once every segment has been processed. +//! +//! # Settled and provisional coverage +//! +//! The last shard in the map may be a provisional tail revision, which will be +//! superseded as the chain advances. Coverage taken from one is reported +//! separately from settled coverage, with the revision's digest, so a caller +//! can re-derive that range instead of merging a later revision into it. +//! +//! # What the checks here establish +//! +//! The first profile trusts the publisher and the retrieval service to provide +//! correct, complete, mutually consistent data for the declared anchor. What +//! this code checks — filter digests against the map, exact script bytes +//! against the row, a page header against the entry that located it, the event +//! total against what the pages yielded — detects corruption, stale data and +//! accidentally mixed shards. None of it establishes that the index matches the +//! chain: a service that returns a consistent lie passes every one of them. +//! +//! # What this leaks, stated plainly +//! +//! Step 3 queries only the shards that matched, so the service learns which +//! chain ranges the wallet had probable activity in. That is the leak the +//! design accepts, and it is real: with content-sealed shards a busy period is +//! covered by a narrower shard, so the localization is sharper during busy +//! periods than quiet ones. This code does not hide it and must not be +//! described as if it did. + +use crate::adapters::{Acceptance, ChainView, ScriptProvider}; +use crate::client::{classify_transport, ClientError, Table, TableClient}; +use crate::ledger::{Ledger, LedgerError}; +use crate::store::{ + uncovered, Anchor, CachedRun, CoverageKind, CoverageRange, PendingPages, RunCursor, + SetIdentity, SetupBlob, SetupKey, ShardCommit, StoreError, StoredEvent, WalletStore, +}; +use crate::transport::ShardRequest; +use crate::transport::{ByteCharges, FilterSource, ShardTransport, StaleRevision}; +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use transparent_events::TransparentEvent; +use transparent_filter::{BlockHash, FilterLimits, ScriptBytes, ShardKey, ShardMap}; +use transparent_shard::build::candidate_rows; +use transparent_shard::manifest::{PageChild, ShardManifest}; +use transparent_shard::page_row::PageEntry; +use transparent_shard::records::DirectoryEntry; + +#[path = "sync_ahead.rs"] +mod ahead; + +#[derive(Debug, thiserror::Error)] +pub enum SyncError { + #[error("client: {0}")] + Client(#[from] ClientError), + #[error("ledger: {0}")] + Ledger(#[from] LedgerError), + #[error("filter: {0}")] + Filter(#[from] transparent_filter::FilterError), + #[error("record: {0}")] + Record(#[from] transparent_shard::records::RecordError), + #[error("transport: {0}")] + Transport(String), + #[error("store: {0}")] + Store(#[from] StoreError), + /// The store's accepted anchor lies above the offered map's end, and the + /// wallet's chain still accepts it. A lower anchor is never accepted + /// silently. + #[error("the store's anchor {stored} is above the map's end {offered}, which the chain still accepts")] + AnchorRegressed { stored: u64, offered: u64 }, + #[error("{0}")] + Invalid(String), + /// A row in a private reply failed on content: it did not decode, a + /// record or link in it was not what the shard declares, or it + /// contradicts what the store holds. A reply failing as a whole (its + /// length, revision binding or parameter epoch) is a request failure + /// instead, [`SyncError::Client`]: the service makes those without + /// knowing the row. + /// + /// A sync does not stop for one: it fails the directory entry the reply + /// was for, finishes everything else, counts the failure in the store + /// and reports [`IncompleteReason::ContentFailed`]; see + /// [`retrieve_shard_into`] and [`walk_chains`]. It reaches a caller from + /// the one-shot [`sync`], which has no store to count it in, and when a + /// contradiction cannot be laid at one entry's door. + #[error("content: {0}")] + Content(String), + #[error("service serves schema {served}, this build reads {expected}")] + Schema { + served: String, + expected: &'static str, + }, + /// A shard names a geometry this build has no parameters for. + /// + /// Fatal for the whole sync rather than a reason to skip the shard. Skipping + /// would advance coverage past a range whose history was never retrieved, + /// and the wallet would report a synchronised balance it has not earned. + #[error( + "shard set uses geometry {0}, which this build does not know; upgrade before syncing this \ + range" + )] + UnknownGeometry(String), + /// The map names a range-filter profile this build does not know. + /// + /// Its filters would be decoded under the wrong Golomb-Rice parameters, + /// which neither fails reliably nor matches correctly, so the sync stops + /// before matching anything. + #[error( + "shard set uses range-filter profile {0}, which this build does not know; upgrade \ + before syncing this range" + )] + UnknownProfile(String), + /// A revision this sync needed was withdrawn, and refreshing the map did + /// not offer one that is served. + /// + /// Terminal, and coverage stopped where it was. Distinct from a transport + /// failure so a caller can re-validate a map against its accepted chain and + /// come back, rather than treating a routine republication as a broken set. + #[error( + "shard {shard_id}: {stale}; the map was refreshed {refreshes} time(s) without offering a \ + live revision" + )] + StaleRevision { + shard_id: u64, + stale: StaleRevision, + refreshes: u32, + }, + /// A shard's manifest does not describe the shard the map named. + /// + /// Terminal for the sync, and coverage stops where it was. Every field a + /// retrieval relies on — geometry, range, segment shape, chain link — is + /// checked against the map before a single private request, so a + /// publication that is internally inconsistent is refused rather than read + /// under whichever of its two stories the wallet happened to see first. + #[error("shard {shard_id}: manifest {field} does not agree with the map")] + ManifestMismatch { shard_id: u64, field: &'static str }, + /// A refreshed map is not the same shard set, continued. + /// + /// The caller validated the map this sync started from against the wallet's + /// accepted chain. A map fetched mid-sync inherits that validation only over + /// the range the two agree on, so a disagreement about already-covered + /// history is a different set rather than a longer one, and the caller has + /// to re-validate before retrying. + #[error( + "the refreshed shard map is not a continuation of the one this sync started from: {0}" + )] + MapDiverged(String), + /// The map contradicts sealed history the store holds on the wallet's own + /// chain, and declares no re-cut of it. + /// + /// Sealed content is immutable. A sealed range described differently is a + /// reorg when the chain has moved, and is rolled back as one. This is the + /// other case: the chain accepts the block the stored range rests on and + /// also the block the map's sealed shard now covering its start ends on, + /// at or below the target. Both are on the chain the wallet follows, so + /// nothing explains the change but the publisher, and retrying does not + /// resolve it while the publisher serves this history. When the chain + /// does not yet accept the replacement's end block, as when the publisher + /// followed a shallow reorg through a shard it had just sealed before the + /// wallet's chain did, the sync stops with + /// [`IncompleteReason::ChainUnknown`] instead and becomes an ordinary reorg + /// once the chain moves. Refused before anything is read, and after + /// rolling back only a reorg the wallet's chain itself shows, which runs + /// first so that no map can hold it back. Distinct from + /// [`MapDiverged`](Self::MapDiverged), which a re-cut published mid-sync + /// also produces and which the next sync resolves. + /// + /// A replica still serving a map from before a re-cut the store has + /// already followed looks the same from here, since the store keeps no + /// re-cut epoch: an embedding that records the epoch it last synced at + /// can tell the two apart before calling the sync. + #[error( + "the map replaces the sealed shard the store holds from height {start_height} (revision \ + {revision_digest}) with another the wallet's chain also accepts, and declares no re-cut \ + of it" + )] + SealedRewrite { + start_height: u64, + revision_digest: String, + }, +} + +/// Map refreshes one sync will spend recovering a withdrawn revision. +/// +/// A tail is republished on a block cadence, so a wallet lapped this many times +/// while reading is not converging on the tip, and saying so beats re-reading a +/// moving target indefinitely. Every refresh must also *change* the refused +/// revision, so this bounds genuine republication rather than a loop. +const MAX_MAP_REFRESHES: u32 = 4; + +/// Passes a sync makes while the wallet's rules keep adding scripts. A +/// gap-limit advance that never converges is a wallet fault; this stops it +/// from becoming an unbounded sync. +const MAX_DISCOVERY_PASSES: u32 = 16; + +/// Content failures after which a directory entry is no longer queried. +/// +/// A service can aim a bad reply at chosen rows and watch what the wallet +/// does next. Each failure changes later traffic once (the entry is asked +/// again, then not), so this is how many such observations an entry gives +/// away; a service could deny the entry service anyway by corrupting it. +pub const MAX_CONTENT_FAILURES: u32 = 3; + +/// One geometry's parameters, as a service declares them. +/// +/// The dimensions come with the schemes so a wallet can check the pair rather +/// than adopt either. A service naming a registry geometry but publishing other +/// row counts under it is refused; so is one publishing the registry's row +/// counts with a scheme they do not derive. +#[allow(dead_code)] // Published compatibility fields are retained for source parity. +pub struct GeometryParams { + pub name: String, + pub directory_rows: u64, + pub directory_row_bytes: u32, + pub directory_scheme: transparent_native::NativeScheme, + /// The 44-bit dithered directory scheme, when the service advertises one. + pub directory_scheme_dq44: Option, + pub directory_setup_seed: u64, + pub page_rows: u64, + pub page_row_bytes: u32, + pub pages_scheme: transparent_native::NativeScheme, + /// The 44-bit dithered page scheme, when the service advertises one. + pub pages_scheme_dq44: Option, + pub pages_setup_seed: u64, +} + +/// The geometries a service declares, checked against this build's registry. +pub struct ServiceGeometry { + /// The schema the service serves, from its init response. + /// + /// A row's meaning is entirely a function of its schema, and the bytes + /// carry no version of their own — a fixed-width row has nowhere to put + /// one without spending space on every row. So a wallet that decoded + /// whatever it was handed would read a newer layout as the one it knows + /// and reconstruct a plausible, wrong history. This is where that stops. + pub schema: String, + /// One entry per geometry the service holds. A set spanning an archive tier + /// and a recent tier declares two. + pub geometries: Vec, +} + +/// The prepared clients for one geometry: one per table. +struct GeometryClients { + directory: TableClient, + pages: TableClient, +} + +/// Every geometry a sync has prepared, by registry name. +/// +/// Prepared on first use rather than up front, so a wallet syncing only recent +/// history never pays to derive archive parameters. Once prepared, a geometry's +/// clients are reused for every shard naming it — which is the saving that +/// naming geometries from a closed registry exists to buy. +struct Clients { + schema: String, + prepared: HashMap, +} + +impl Clients { + /// Prepares the clients for `name`, checking the service's declaration + /// against this build's registry and re-deriving the scheme. + /// + /// Three things have to agree: the name must be one this build knows, the + /// dimensions the service published under it must be the registry's, and + /// the scheme must be what those dimensions derive. A service that could + /// move any one of them independently could choose a geometry for the + /// wallet, and choosing the geometry is choosing what the response reveals. + fn prepare( + &mut self, + name: &str, + declared: &[GeometryParams], + ) -> Result<&mut GeometryClients, SyncError> { + if !self.prepared.contains_key(name) { + let geometry = transparent_shard::layout::by_name(name) + .ok_or_else(|| SyncError::UnknownGeometry(name.to_string()))?; + let params = declared + .iter() + .find(|params| params.name == name) + .ok_or_else(|| { + SyncError::Invalid(format!( + "the map uses geometry {name} but the service declares no parameters for it" + )) + })?; + if params.directory_rows != geometry.directory_rows + || params.directory_row_bytes != geometry.directory_row_bytes as u32 + || params.page_rows != geometry.page_rows + || params.page_row_bytes != geometry.page_row_bytes as u32 + { + return Err(SyncError::Invalid(format!( + "the service declares geometry {name} with dimensions this build does not \ + know it by" + ))); + } + let clients = GeometryClients { + directory: TableClient::new_with_schema( + &self.schema, + Table::Directory, + name, + geometry.directory_rows, + geometry.directory_row_bytes as u32, + ¶ms.directory_scheme, + params.directory_scheme_dq44.as_ref(), + )?, + pages: TableClient::new_with_schema( + &self.schema, + Table::Pages(0), + name, + geometry.page_rows, + geometry.page_row_bytes as u32, + ¶ms.pages_scheme, + params.pages_scheme_dq44.as_ref(), + )?, + }; + self.prepared.insert(name.to_string(), clients); + } + Ok(self.prepared.get_mut(name).expect("just prepared")) + } +} + +/// One provisional revision a sync took coverage from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProvisionalCoverage { + pub shard_id: u64, + pub revision: u32, + pub manifest_digest: String, + pub end_height: u64, +} + +/// Bounds on one call to [`sync_into`]. `None` is unbounded. +/// +/// Reaching a bound is not an error: the sync commits what it has, records +/// the page work still owed, and reports `Incomplete`. The next call resumes +/// from the durable pending work. +#[derive(Clone, Copy, Debug, Default)] +pub struct WorkLimits { + /// Private queries, directory and pages together. + pub max_queries: Option, + /// Private bytes: setup, query uploads and responses. + pub max_private_bytes: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IncompleteReason { + QueryBudget, + ByteBudget, + /// The store would not hold more pending page work. + PendingLimit, + /// The service stayed at capacity for the whole retry budget on this shard. + Overloaded { + shard_id: u64, + }, + /// The wallet's chain view could not confirm a block its coverage rests + /// on, so neither a reorg nor its absence could be established. + ChainUnknown { + height: u64, + }, + /// The wallet's rules kept adding scripts past the pass bound. + DiscoveryUnbounded, + /// Coverage stops short at directory entries whose replies failed on + /// content; [`SyncReport::failed_entries`] lists them. Everything else + /// was read and committed. + ContentFailed, + PublicationBehind { + height: u64, + }, + UnresolvedSpends, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Completion { + Complete, + Incomplete { + reason: IncompleteReason, + /// Pending page retrievals left in the store. + pending: usize, + }, +} + +/// What one [`sync_into`] call did. +#[allow(dead_code)] // Internal parity report, never a product balance API. +pub struct SyncReport { + /// The store's whole ledger after this sync, not only what it added. + pub ledger: Ledger, + pub charges: ByteCharges, + pub matched_shards: Vec, + pub unproductive_matches: u64, + /// Height through which every script in scope is covered from its + /// required height, provisional coverage included. + pub covered_through: u64, + /// The same counting settled coverage only. + pub settled_through: u64, + pub provisional: Vec, + pub map_refreshes: u32, + pub completion: Completion, + /// The height everything above was rolled back to, if this sync found a + /// reorg or a replaced provisional tail. + pub rolled_back_to: Option, + /// Provisional revision digests whose coverage was truncated and + /// re-derived. + pub replaced_revisions: Vec, + /// Scripts the wallet's rules added during this sync. + pub scripts_added: usize, + /// Store commits this sync made. + pub commits: u64, + /// Directory entries this sync failed on content or held back because + /// they had failed [`MAX_CONTENT_FAILURES`] times, by shard and script. + pub failed_entries: Vec, +} + +/// One script's directory entry in one shard that failed on content. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FailedEntry { + pub shard_id: u64, + pub script: Vec, + /// Content failures the store holds for the entry, this sync's included. + /// At [`MAX_CONTENT_FAILURES`] the entry is no longer queried. + pub failures: u32, +} + +/// Syncs a wallet's store to its accepted target, independently of publication tip. +/// +/// See the module documentation for the shape of a sync. The store is the +/// wallet's memory: what it holds decides what is fetched, and what is +/// fetched is committed to it shard by shard. On return the store is +/// consistent whether the sync completed or not. +#[allow(clippy::too_many_arguments)] +pub async fn sync_into( + store: &mut S, + map: &ShardMap, + map_bytes: u64, + geometry: &ServiceGeometry, + chain: &impl ChainView, + scripts: &mut impl ScriptProvider, + filters: &mut impl FilterSource, + transport: &mut impl ShardTransport, + limits: &WorkLimits, + target_anchor: &Anchor, +) -> Result { + if !transparent_shard::manifest::supported_schema(&geometry.schema) { + return Err(SyncError::Schema { + served: geometry.schema.clone(), + expected: transparent_shard::SCHEMA, + }); + } + map.check_shape() + .map_err(|error| SyncError::Invalid(format!("shard map is malformed: {error}")))?; + if transparent_filter::range_profile(&map.profile).is_none() { + return Err(SyncError::UnknownProfile(map.profile.clone())); + } + let genesis = BlockHash::from_display_hex(&map.genesis_hash)?; + BlockHash::from_display_hex(&target_anchor.hash)?; + if target_anchor.height < map.start_height { + return Err(SyncError::Invalid( + "target precedes publication start".into(), + )); + } + match chain.is_accepted(target_anchor.height, &target_anchor.hash) { + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "target anchor rejected by wallet chain".into(), + )) + } + Acceptance::Unknown => { + return incomplete_at( + store, + IncompleteReason::ChainUnknown { + height: target_anchor.height, + }, + ) + } + Acceptance::Accepted => {} + } + if map + .shards + .last() + .is_none_or(|s| s.end_height < target_anchor.height) + { + return incomplete_at( + store, + IncompleteReason::PublicationBehind { + height: target_anchor.height, + }, + ); + } + check_target_hash(map, target_anchor)?; + let retained = store + .events()? + .iter() + .map(|e| event_height(&e.event)) + .max() + .unwrap_or(0); + if let Some(a) = store.anchor()? { + if a.height > target_anchor.height + && chain.is_accepted(a.height, &a.hash) == Acceptance::Accepted + { + return Err(SyncError::AnchorRegressed { + stored: a.height, + offered: target_anchor.height, + }); + } + } + if retained > target_anchor.height { + return Err(SyncError::Invalid( + "target below retained events; explicitly roll back to an accepted anchor first".into(), + )); + } + + // The store is bound to a publication lineage. A map from another + // lineage is refused before anything is read from it. + let identity = SetIdentity::of_schema(map, &geometry.schema); + store.bind_set(&identity).map_err(|error| match error { + StoreError::SetMismatch { stored, offered } => SyncError::MapDiverged(format!( + "the store is bound to set {stored}, this map is {offered}" + )), + other => SyncError::Store(other), + })?; + let first_commit = store.last_commit()?; + + let Some(tip_entry) = map.shards.last() else { + return Err(SyncError::Invalid("the map names no shards".into())); + }; + let through = target_anchor.height; + let _ = tip_entry; + + // A map that ends below the anchor the wallet accepted is not a newer + // publication; if the wallet's chain still accepts that anchor, the + // map is refused rather than silently rewound. + if let Some(anchor) = store.anchor()? { + if anchor.height > through + && chain.is_accepted(anchor.height, &anchor.hash) == Acceptance::Accepted + { + return Err(SyncError::AnchorRegressed { + stored: anchor.height, + offered: through, + }); + } + } + + let mut charges = ByteCharges { + map_bytes, + ..Default::default() + }; + let mut rolled_back_to: Option = None; + let mut replaced_revisions: Vec = Vec::new(); + + // Every revision the map's re-cuts declare, by digest, looked up once. + let declared = map.superseded_index(); + + // Reorg detection over what the store holds. Every distinct block the + // coverage rests on is asked of the wallet's chain, newest first; the + // first rejected one rolls everything above the highest accepted block + // below it back. + let mut rests_on: BTreeMap = BTreeMap::new(); + // Settled ranges the map neither still publishes nor declares superseded. + let mut rewritten: Vec = Vec::new(); + for entry in store.scripts()? { + for range in store.coverage(&entry.script)? { + rests_on + .entry(range.end_height) + .or_insert(range.terminal_block_hash.clone()); + if range.kind == CoverageKind::Settled && !vouches_for(map, &declared, &range) { + rewritten.push(range); + } + } + } + let mut ancestor: Option = None; + let mut unknown: Option = None; + for (height, hash) in rests_on.iter().rev() { + match chain.is_accepted(*height, hash) { + Acceptance::Rejected => { + // Roll back to the highest accepted block below this one. + let mut below = map.start_height.saturating_sub(1); + for (candidate, candidate_hash) in rests_on.range(..height).rev() { + if chain.is_accepted(*candidate, candidate_hash) == Acceptance::Accepted { + below = *candidate; + break; + } + } + ancestor = Some(below); + break; + } + Acceptance::Accepted => break, + Acceptance::Unknown => { + unknown = Some(*height); + } + } + } + if let Some(height) = ancestor { + store.rollback_above(&accepted_at(chain, map, height)?, "reorg")?; + rolled_back_to = Some(height); + } else if let Some(height) = unknown { + if !rests_on.is_empty() { + // Nothing could be confirmed. Reading on would either accept a + // branch the wallet has not seen or roll back on a guess. + return chain_unknown( + store, + map, + target_anchor, + charges, + height, + rolled_back_to, + replaced_revisions, + first_commit, + ); + } + } + + // Sealed content is immutable, so a sealed range the map now describes + // differently, without declaring a re-cut of it, is judged by what the + // wallet's chain says; see `judge_rewrite`. Only after the scan above, + // which depends on the chain alone, has rolled back any reorg the chain + // shows, and only for the ranges that rollback kept: a map cannot hold + // back a reorg by changing history below it. A contradiction on the + // wallet's own chain is refused before anything else is rolled back or + // read; anything the chain cannot settle yet stops the sync as an + // unknown block does. + let mut contradiction: Option<&CoverageRange> = None; + let mut unplaced: Option = None; + for range in rewritten + .iter() + .filter(|range| ancestor.is_none_or(|kept| range.end_height <= kept)) + { + match judge_rewrite(map, range, target_anchor, chain) { + Rewrite::Reorg => {} + Rewrite::Contradiction => { + if contradiction.is_none_or(|held| held.start_height > range.start_height) { + contradiction = Some(range); + } + } + Rewrite::Unsettled(height) => { + unplaced = Some(unplaced.map_or(height, |held| held.max(height))); + } + } + } + if let Some(range) = contradiction { + return Err(SyncError::SealedRewrite { + start_height: range.start_height, + revision_digest: range.revision_digest.clone(), + }); + } + if let Some(height) = unplaced { + return chain_unknown( + store, + map, + target_anchor, + charges, + height, + rolled_back_to, + replaced_revisions, + first_commit, + ); + } + + // Provisional reconciliation. A tail revision the map has moved past is + // truncated and re-derived; one the map has sealed is promoted. + let mut truncate_from: Option = None; + let mut promote: Vec<(u64, String)> = Vec::new(); + // A partially retrieved revision may have events and pending pages but no + // completed coverage yet. Unfinished work under a revision the map no + // longer publishes is dropped, and what it saved is rolled back from + // where the revision began, never from wherever the map now puts its + // shard id: a re-cut gives that id to another range. + let mut dropped: Vec = Vec::new(); + let mut saved: Option> = None; + for pending in store.pending()? { + if map + .shards + .get(pending.shard_id as usize) + .is_some_and(|published| published.manifest_digest == pending.revision_digest) + { + continue; + } + if saved.is_none() { + saved = Some(store.events()?); + } + let lowest_saved = saved + .iter() + .flatten() + .filter(|stored| stored.revision_digest == pending.revision_digest) + .map(|stored| event_height(&stored.event)) + .min(); + let declared = declared.get(pending.revision_digest.as_str()).copied(); + match unfinished(&pending, declared, lowest_saved, chain) { + Unfinished::Rewind(height) => { + truncate_from = Some(truncate_from.map_or(height, |held| held.min(height))); + } + Unfinished::Drop => dropped.extend(pending.id), + } + if !replaced_revisions.contains(&pending.revision_digest) { + replaced_revisions.push(pending.revision_digest); + } + } + for range in store.provisional()? { + match map.shards.get(range.shard_id as usize) { + Some(published) if published.manifest_digest == range.revision_digest => { + if published.sealed { + promote.push((range.shard_id, range.revision_digest.clone())); + } + } + _ => { + truncate_from = Some( + truncate_from + .map_or(range.start_height, |held: u64| held.min(range.start_height)), + ); + if !replaced_revisions.contains(&range.revision_digest) { + replaced_revisions.push(range.revision_digest.clone()); + } + } + } + } + if let Some(start) = truncate_from { + // From the start of the shard now covering that height: at or below + // everything the replaced revision saved, and a boundary the map names + // a block for, so the rollback never depends on the wallet's chain + // holding a hash inside a shard. + let start = map + .shard_for_height(start) + .map_or(start, |entry| entry.start_height.min(start)); + let below = start.saturating_sub(1); + if rolled_back_to.is_none_or(|held| held > below) { + store.rollback_above(&accepted_at(chain, map, below)?, "replaced revision")?; + rolled_back_to = Some(below); + } + } + for (shard_id, digest) in promote { + store.promote_provisional(shard_id, &digest)?; + } + + let discard: Vec = store + .pending()? + .iter() + .filter(|p| { + (p.target_anchor.as_ref() != Some(target_anchor) && p.dummy_debt.is_none()) + || p.id.is_some_and(|id| dropped.contains(&id)) + }) + .filter_map(|p| p.id) + .collect(); + if !discard.is_empty() { + store.commit_shard(ShardCommit { + pending_complete: discard, + ..Default::default() + })?; + } + + // Replies a lookahead sends early are held here until the walk asks for + // them; see `ahead`. With nothing stashed both pass straight through. + let transport = &mut ahead::Staged::new(transport); + let filters = &mut ahead::StagedFilters::new(filters); + // A budgeted sync walks in sequence: whether a later shard's queries are + // sent at all depends on page work the walk discovers on the way. + let look_ahead = transport.concurrency() > 1 + && limits.max_queries.is_none() + && limits.max_private_bytes.is_none(); + + // The wallet's script set, with its own required heights. + store.add_scripts(&scripts.scripts())?; + let mut scripts_added = 0usize; + let mut clients = Clients { + schema: geometry.schema.clone(), + prepared: HashMap::new(), + }; + let mut matched_shards: Vec = Vec::new(); + let mut unproductive = 0u64; + let mut refreshes = 0u32; + let mut held_map_digest: Option = None; + let mut active: Cow<'_, ShardMap> = Cow::Borrowed(map); + let mut completion = Completion::Complete; + let mut failures = Failures::default(); + // A cold pass must retain both the cover obligation and the failed-entry + // hold until the obligation is paid; otherwise it could query the real head. + for item in store + .pending()? + .iter() + .filter(|item| item.dummy_debt.is_some()) + { + record_failure(store, &mut failures.entries, item.shard_id, &item.script)?; + } + + 'passes: for pass in 0..=MAX_DISCOVERY_PASSES { + if pass == MAX_DISCOVERY_PASSES { + completion = Completion::Incomplete { + reason: IncompleteReason::DiscoveryUnbounded, + pending: store.pending()?.len(), + }; + break; + } + let entries = store.scripts()?; + let mut active_scripts: Vec> = Vec::new(); + + // Pending page work first: it is already located, and finishing it + // is the cheapest coverage available. + let pending = store.pending()?; + let mut by_revision: BTreeMap<(u64, String), Vec> = BTreeMap::new(); + for item in pending { + by_revision + .entry((item.shard_id, item.revision_digest.clone())) + .or_default() + .push(item); + } + for ((shard_id, digest), mut items) in by_revision { + // An entry that failed earlier in this sync, or too often, is + // not walked again; one still owing dummies sends them. + let mut kept = Vec::with_capacity(items.len()); + for item in items { + if !hold(store, &mut failures.entries, shard_id, &item.script)? + || item.dummy_debt.is_some() + { + kept.push(item); + } + } + items = kept; + if items.is_empty() { + continue; + } + let Some(entry) = active.shards.get(shard_id as usize) else { + continue; + }; + if entry.manifest_digest != digest { + // Its revision is gone, here only after a refresh mid-sync; + // the ordinary planning re-derives the range. + continue; + } + let entry = entry.clone(); + let previous_digest = previous_digest_of(&active, shard_id); + let outcome = with_refusals( + store, + &active, + &entry, + &previous_digest, + geometry, + &mut clients, + transport, + &mut charges, + limits, + async |store, prepared, transport, charges, manifest| { + finish_pages( + store, + &entry, + manifest, + &items, + prepared, + transport, + charges, + limits, + target_anchor, + chain, + &mut failures, + ) + .await + }, + ) + .await; + match outcome { + Ok(Some(stopped)) => { + completion = stopped; + break 'passes; + } + Ok(None) => { + if !matched_shards.contains(&shard_id) { + matched_shards.push(shard_id); + } + let events = store.events()?; + for item in &items { + if holds_history(&events, &item.script, &entry, target_anchor.height) + && !active_scripts.contains(&item.script) + { + active_scripts.push(item.script.clone()); + } + } + } + Err(SyncError::StaleRevision { .. }) => { + // Handled by the ordinary planning below on the next map. + } + Err(other) => return Err(other), + } + } + + // Plan: which shards each script still needs. + let through = target_anchor.height; + let mut work: BTreeMap>> = BTreeMap::new(); + for entry in &entries { + let from = entry.required_from.max(active.start_height); + if from > through { + continue; + } + let coverage = store.coverage(&entry.script)?; + for (gap_start, gap_end) in uncovered(&coverage, from, through) { + for (index, shard) in active.shards.iter().enumerate() { + if shard.end_height < gap_start { + continue; + } + if shard.start_height > gap_end { + break; + } + let scripts = work.entry(index).or_default(); + if !scripts.contains(&entry.script) { + scripts.push(entry.script.clone()); + } + } + } + } + + let mut uncached = Vec::new(); + for index in work.keys() { + let entry = &active.shards[*index]; + if store + .filter(&entry.manifest_digest, &entry.filter_hash)? + .is_none() + { + uncached.push(entry.shard_id); + } + } + if filters.uses_parents() { + charges.filter_bytes += filters + .prepare_parents(&active, &uncached, store) + .await + .map_err(|e| SyncError::Transport(e.to_string()))?; + } else { + // Every uncached filter in the walk is downloaded anyway; overlap + // them rather than paying one round trip each. + filters.prefetch(&uncached); + } + let mut index_iter: Vec = work.keys().copied().collect(); + if look_ahead && !filters.uses_parents() { + ahead::look_ahead( + store, + &active, + genesis, + &index_iter, + &work, + geometry, + &mut clients, + filters, + transport, + target_anchor, + chain, + &failures.entries, + ) + .await?; + } + let mut position = 0usize; + while position < index_iter.len() { + let index = index_iter[position]; + let Some(entry) = active.shards.get(index).cloned() else { + break; + }; + let scripts_here = work.get(&index).cloned().unwrap_or_default(); + let previous_digest = previous_digest_of(&active, entry.shard_id); + let result = read_shard_into( + store, + &active, + &entry, + &previous_digest, + genesis, + &scripts_here, + geometry, + &mut clients, + filters, + transport, + &mut charges, + limits, + target_anchor, + chain, + &mut failures, + ) + .await; + match result { + Ok(ShardRead::Done { matched }) => { + if !matched.is_empty() { + if !matched_shards.contains(&entry.shard_id) { + matched_shards.push(entry.shard_id); + } + for script in matched { + if !active_scripts.contains(&script) { + active_scripts.push(script); + } + } + } + position += 1; + } + Ok(ShardRead::Unproductive { count }) => { + unproductive += count; + position += 1; + } + Ok(ShardRead::Stopped(stopped)) => { + completion = stopped; + break 'passes; + } + Err(SyncError::StaleRevision { stale, .. }) => { + // The service withdrew this revision while the sync was + // reading it. Refresh the map, check it is the same set + // continued, roll this shard's provisional coverage back, + // and re-derive from whatever replaced it. + // + // Nothing read ahead survives a refresh: the rest of this + // pass is walked in sequence. + ahead::clear(transport, filters, &mut clients); + if refreshes >= MAX_MAP_REFRESHES + || (stale.map_sha256.is_some() + && stale.map_sha256.as_deref() == held_map_digest.as_deref()) + { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes, + }); + } + let (covered, _) = coverage_summary(store, &active, target_anchor.height)?; + let (fresh, digest) = + refresh_map(&active, covered, filters, &mut charges).await?; + check_target_hash(&fresh, target_anchor)?; + if fresh + .shards + .last() + .is_none_or(|s| s.end_height < target_anchor.height) + { + completion = Completion::Incomplete { + reason: IncompleteReason::PublicationBehind { + height: target_anchor.height, + }, + pending: store.pending()?.len(), + }; + active = Cow::Owned(fresh); + break 'passes; + } + refreshes += 1; + held_map_digest = Some(digest); + let Some(resume) = fresh + .shards + .iter() + .position(|candidate| candidate.shard_id == entry.shard_id) + else { + // The refreshed map no longer advertises this range. + active = Cow::Owned(fresh); + break; + }; + if fresh.shards[resume].manifest_digest == stale.revision { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes, + }); + } + check_resume(&fresh.shards[resume], &entry).map_err(SyncError::MapDiverged)?; + let resume_start = fresh.shards[resume].start_height; + let below = resume_start.saturating_sub(1); + store.rollback_above( + &accepted_at(chain, &active, below)?, + "revision withdrawn mid-sync", + )?; + if rolled_back_to.is_none_or(|held| held > below) { + rolled_back_to = Some(below); + } + if !replaced_revisions.contains(&stale.revision) { + replaced_revisions.push(stale.revision.clone()); + } + active = Cow::Owned(fresh); + // Re-plan against the new map from this shard on. + let mut rest: Vec = (resume..active.shards.len()) + .take_while(|i| active.shards[*i].start_height <= through) + .collect(); + index_iter.truncate(position); + index_iter.append(&mut rest); + for later in resume..active.shards.len() { + work.entry(later).or_insert_with(|| scripts_here.clone()); + } + } + Err(other) => return Err(other), + } + } + // Anything read ahead and not consumed belongs to this pass only. + ahead::clear(transport, filters, &mut clients); + + let added = scripts.on_activity(&active_scripts); + if added.is_empty() { + break; + } + scripts_added += store.add_scripts(&added)?; + } + + let (covered_through, settled_through) = + coverage_summary(store, &active, target_anchor.height)?; + if completion == Completion::Complete { + if covered_through < target_anchor.height { + // A failed entry leaves its script uncovered over its shard. + completion = Completion::Incomplete { + reason: if failures.entries.is_empty() { + IncompleteReason::DiscoveryUnbounded + } else { + IncompleteReason::ContentFailed + }, + pending: store.pending()?.len(), + }; + } else if !store.pending()?.is_empty() { + completion = Completion::Incomplete { + reason: IncompleteReason::PendingLimit, + pending: store.pending()?.len(), + }; + } else if !store.ledger()?.unresolved().is_empty() { + completion = Completion::Incomplete { + reason: IncompleteReason::UnresolvedSpends, + pending: 0, + }; + } else { + store.commit_anchor(target_anchor, settled_through, covered_through)?; + } + } + matched_shards.sort_unstable(); + let mut failed = failures.entries; + failed.sort_by(|a, b| (a.shard_id, &a.script).cmp(&(b.shard_id, &b.script))); + Ok(SyncReport { + ledger: store.ledger()?, + charges, + matched_shards, + unproductive_matches: unproductive, + covered_through, + settled_through, + provisional: provisional_of(store, &active)?, + map_refreshes: refreshes, + completion, + rolled_back_to, + replaced_revisions, + scripts_added, + commits: store.last_commit()? - first_commit, + failed_entries: failed, + }) +} + +/// Whether the map still vouches for a settled range the store holds. +/// +/// The range is looked up by the height it starts at, not by its shard id: a +/// re-cut renumbers shards, so the id it was read under may now name another +/// range. The map vouches for it when it still publishes that revision there, +/// or when one of its re-cuts declares exactly that revision superseded: the +/// same digest, shard id and start height, sealed, ending on the same block. +/// That keeps a wallet's history across the re-cut without re-reading it. +/// +/// The block a range ends on is its source anchor, the published endpoint, +/// when the store kept one, which every store since schema 2 does. A range +/// without one matches a declaration when it ends inside the declared range, +/// or at its end on the declared block. A range with an endpoint but cut +/// short of its shard's end by a target or a rollback is left to the chain +/// checks, as is anything the map's sealed shards do not cover. +fn vouches_for( + map: &ShardMap, + declared: &BTreeMap<&str, &transparent_filter::SupersededShard>, + range: &CoverageRange, +) -> bool { + if range + .source_anchor + .as_ref() + .is_some_and(|source| source.height != range.end_height) + { + return true; + } + let Some(entry) = map.shard_for_height(range.start_height) else { + return true; + }; + if !entry.sealed { + return true; + } + if entry.start_height == range.start_height + && entry.manifest_digest == range.revision_digest + && range.source_anchor.as_ref().is_none_or(|source| { + source.height == entry.end_height && source.hash == entry.terminal_block_hash + }) + { + return true; + } + declared + .get(range.revision_digest.as_str()) + .is_some_and(|declared| { + let ends_as_declared = match &range.source_anchor { + Some(source) => { + declared.end_height == source.height + && declared.terminal_block_hash == source.hash + } + // Without a recorded endpoint a range may have been cut short + // of its shard by a target or a rollback, as one with an + // endpoint is above: inside the declared range it matches, + // and at the declared end it must rest on the declared block. + None => { + range.end_height < declared.end_height + || (range.end_height == declared.end_height + && range.terminal_block_hash == declared.terminal_block_hash) + } + }; + declared.sealed + && declared.shard_id == range.shard_id + && declared.start_height == range.start_height + && ends_as_declared + }) +} + +/// What a settled range the map no longer vouches for means. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Rewrite { + /// The wallet's chain rejects the block the range rests on: a reorg, + /// rolled back like any other. + Reorg, + /// The wallet's chain accepts both the block the range rests on and the + /// one the shard now covering its start ends on: the publisher + /// contradicts itself on the chain the wallet follows. Retrying does not + /// change that while it serves this history. + Contradiction, + /// The chain cannot settle it yet; the height is the block it would need + /// to confirm. A publisher that followed a shallow reorg across a shard it + /// had just sealed, before the wallet's chain did, looks like this, and + /// becomes an ordinary reorg once the chain moves. + Unsettled(u64), +} + +/// Judges a settled range `vouches_for` refused, by what the wallet's chain +/// says about the block it rests on and about the block the map's shard now +/// covering its start ends on. +/// +/// Sealing is not delayed for finality, so a publisher can follow a reorg +/// through a just-sealed shard before the wallet's chain does. Only when the +/// chain accepts the replacement's end too, at or below the target, is the +/// change the publisher's alone. +fn judge_rewrite( + map: &ShardMap, + range: &CoverageRange, + target: &Anchor, + chain: &impl ChainView, +) -> Rewrite { + match chain.is_accepted(range.end_height, &range.terminal_block_hash) { + Acceptance::Rejected => Rewrite::Reorg, + Acceptance::Unknown => Rewrite::Unsettled(range.end_height), + Acceptance::Accepted => match map.shard_for_height(range.start_height) { + Some(entry) + if entry.sealed + && entry.end_height <= target.height + && chain.is_accepted(entry.end_height, &entry.terminal_block_hash) + == Acceptance::Accepted => + { + Rewrite::Contradiction + } + Some(entry) => Rewrite::Unsettled(entry.end_height), + // `vouches_for` refuses only ranges a sealed shard covers, so + // this and the unsealed case above are not reached; both would + // be the publisher ahead of or apart from the chain, not settled. + None => Rewrite::Unsettled(range.end_height), + }, + } +} + +/// What becomes of unfinished page work under a revision the map no longer +/// publishes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Unfinished { + /// Dropped, and what the revision saved stays. + Drop, + /// Dropped, and the store rolls back from this height. + Rewind(u64), +} + +/// Decides [`Unfinished`] for one item, from what the wallet itself stored. +/// +/// Every event an item saved lies at or below the target it was read for, +/// which the store keeps with it. So when a declared re-cut replaced the +/// sealed revision and the wallet's chain still accepts that target block, +/// what was saved is history on the chain the wallet follows, which a re-cut +/// does not change: it stays, the script's gap is read again under the shard +/// now covering it, and events are kept once per outpoint. The declaration +/// only says this is a re-cut; nothing it claims about heights or blocks is +/// relied on. +/// +/// Otherwise what the revision saved is rolled back from the lowest height it +/// saved anything at, or the declared start if that is lower; never from the +/// declared start alone, which the wallet has not checked. A revision that +/// saved nothing leaves nothing to roll back. +fn unfinished( + pending: &PendingPages, + declared: Option<&transparent_filter::SupersededShard>, + lowest_saved: Option, + chain: &impl ChainView, +) -> Unfinished { + let declared = declared.filter(|declared| declared.shard_id == pending.shard_id); + let target_held = pending.target_anchor.as_ref().is_some_and(|target| { + chain.is_accepted(target.height, &target.hash) == Acceptance::Accepted + }); + if declared.is_some_and(|declared| declared.sealed) && target_held { + return Unfinished::Drop; + } + match lowest_saved { + Some(lowest) => Unfinished::Rewind( + declared.map_or(lowest, |declared| declared.start_height.min(lowest)), + ), + None => Unfinished::Drop, + } +} + +/// Whether `script` has stored history inside `entry`'s range, up to the +/// target. +/// +/// By height rather than by the revision that recorded it: an event is kept +/// once per outpoint, so history first read under a revision a re-cut has +/// since replaced keeps that revision's name when the new shard is read. +fn holds_history( + events: &[StoredEvent], + script: &[u8], + entry: &transparent_filter::ShardMapEntry, + through: u64, +) -> bool { + let range = entry.start_height..=entry.end_height.min(through); + events + .iter() + .any(|stored| stored.script == script && range.contains(&event_height(&stored.event))) +} + +fn previous_digest_of(map: &ShardMap, shard_id: u64) -> String { + shard_id + .checked_sub(1) + .and_then(|before| map.shards.get(before as usize)) + .map(|entry| entry.manifest_digest.clone()) + .unwrap_or_default() +} + +/// Heights through which every script is covered from its required height: +/// provisional coverage included, then settled only. The minimum over +/// scripts, so one script's gap holds the whole wallet's figure down. +fn coverage_summary( + store: &S, + map: &ShardMap, + through: u64, +) -> Result<(u64, u64), SyncError> { + let floor = map.start_height.saturating_sub(1); + let mut covered = through; + let mut settled = through; + let entries = store.scripts()?; + if entries.is_empty() { + return Ok((floor, floor)); + } + for entry in entries { + let from = entry.required_from.max(map.start_height); + let ranges = store.coverage(&entry.script)?; + let all = uncovered(&ranges, from, through); + let mine = all + .first() + .map_or(through, |(start, _)| start.saturating_sub(1)); + covered = covered.min(mine); + let settled_ranges: Vec = ranges + .iter() + .filter(|range| range.kind == CoverageKind::Settled) + .cloned() + .collect(); + let settled_gaps = uncovered(&settled_ranges, from, through); + let mine_settled = settled_gaps + .first() + .map_or(through, |(start, _)| start.saturating_sub(1)); + settled = settled.min(mine_settled); + } + Ok((covered.max(floor), settled.max(floor))) +} + +fn provisional_of( + store: &S, + map: &ShardMap, +) -> Result, SyncError> { + let mut seen: BTreeMap<(u64, String), u64> = BTreeMap::new(); + for range in store.provisional()? { + let end = seen + .entry((range.shard_id, range.revision_digest.clone())) + .or_insert(range.end_height); + *end = (*end).max(range.end_height); + } + Ok(seen + .into_iter() + .map( + |((shard_id, manifest_digest), end_height)| ProvisionalCoverage { + shard_id, + // The digest identifies the revision; its number is the map's. + revision: map + .shards + .get(shard_id as usize) + .filter(|entry| entry.manifest_digest == manifest_digest) + .map(|entry| entry.revision) + .unwrap_or(0), + manifest_digest, + end_height, + }, + ) + .collect()) +} + +enum ShardRead { + /// Every script in scope was covered; `matched` names those with history. + Done { matched: Vec> }, + /// The filter matched but nothing was found; coverage advanced anyway. + Unproductive { count: u64 }, + /// A budget or refusal stopped the sync with the store consistent. + Stopped(Completion), +} + +/// Reads one shard for the scripts that need it, committing to the store. +#[allow(clippy::too_many_arguments)] +async fn read_shard_into( + store: &mut S, + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + genesis: BlockHash, + scripts: &[Vec], + geometry: &ServiceGeometry, + clients: &mut Clients, + filters: &mut impl FilterSource, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result { + let endpoint = endpoint_of(entry, target_anchor); + match chain.is_accepted(endpoint.height, &endpoint.hash) { + Acceptance::Accepted => {} + Acceptance::Unknown => { + return Ok(ShardRead::Stopped(Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { + height: endpoint.height, + }, + pending: store.pending()?.len(), + })) + } + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "published coverage endpoint rejected by wallet chain".into(), + )) + } + } + // Ordinary sources download every filter in range. The explicit experiment + // may establish a parent negative; both paths use the same bounded commit. + let cached = store.filter(&entry.manifest_digest, &entry.filter_hash)?; + let (parent_negative, parent_cost) = if cached.is_none() { + filters + .parent_negative(map, entry.shard_id, scripts, store) + .await + .map_err(|e| SyncError::Transport(e.to_string()))? + } else { + (false, 0) + }; + charges.filter_bytes += parent_cost; + let matches = if parent_negative { + Vec::new() + } else { + let bytes = match cached { + Some(bytes) => bytes, + None => { + let (bytes, cost) = filters.filter(entry.shard_id).await.map_err(|error| { + SyncError::Transport(crate::transport::describe_error(error.as_ref())) + })?; + charges.filter_bytes += cost; + // Bind the bytes to the map before believing anything they say. + if transparent_filter::filter_hash(&bytes).to_display_hex() != entry.filter_hash { + // The public filter URL is shard-addressed: activation may + // have replaced it since this map was fetched. Never cache or + // match these bytes. The bounded stale-revision path requires + // a changed, validated map before retrying; an unchanged map + // still fails, including when the filter was simply corrupt. + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale: StaleRevision { + shard_id: entry.shard_id, + revision: entry.manifest_digest.clone(), + map_sha256: None, + }, + refreshes: 0, + }); + } + store.put_filter( + &entry.manifest_digest, + &entry.filter_hash, + entry.sealed, + &bytes, + )?; + bytes + } + }; + charges.filters_checked += 1; + match_filter(map, entry, genesis, scripts, &bytes)? + }; + if matches.is_empty() { + // A genuinely empty range for every script here: coverage advances + // with no private request. + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events: Vec::new(), + covered_scripts: scripts.to_vec(), + pending_upsert: Vec::new(), + pending_complete: Vec::new(), + cached_runs: Vec::new(), + failed_entries: Vec::new(), + }, + )?; + return Ok(ShardRead::Done { + matched: Vec::new(), + }); + } + // A geometry this build does not know is fatal the moment there is + // something to fetch, whatever the manifest would say about it. + if transparent_shard::layout::by_name(&entry.geometry).is_none() { + return Err(SyncError::UnknownGeometry(entry.geometry.clone())); + } + let matched_all: Vec> = matches.iter().map(|&i| scripts[i].clone()).collect(); + let unmatched: Vec> = scripts + .iter() + .filter(|script| !matched_all.contains(script)) + .cloned() + .collect(); + // An entry held out of this sync is not queried at all, and stays + // uncovered over this shard. + let mut matched_scripts = Vec::with_capacity(matched_all.len()); + for script in matched_all { + if !hold(store, &mut failures.entries, entry.shard_id, &script)? { + matched_scripts.push(script); + } + } + if matched_scripts.is_empty() { + if !unmatched.is_empty() { + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + covered_scripts: unmatched, + ..Default::default() + }, + )?; + } + return Ok(ShardRead::Done { + matched: Vec::new(), + }); + } + + let outcome = with_refusals( + store, + map, + entry, + previous_digest, + geometry, + clients, + transport, + charges, + limits, + async |store, prepared, transport, charges, manifest| { + // Validated when the manifest was verified; decoded again here + // rather than carried, since it is a few kilobytes. + let choice = directory_choice(manifest)?; + let salt = tag_salt_of(manifest)?; + retrieve_shard_into( + store, + entry, + manifest, + &matched_scripts, + &unmatched, + choice.as_ref(), + &salt, + prepared, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await + }, + ) + .await?; + match outcome { + Some(stopped) => Ok(ShardRead::Stopped(stopped)), + None => { + // Which matched scripts actually held history is what the + // caller's gap-limit rule wants; the store knows. + let mut found = Vec::new(); + let mut unproductive = 0u64; + let events = store.events()?; + for script in &matched_scripts { + if failures + .entries + .iter() + .any(|held| held.shard_id == entry.shard_id && held.script == *script) + { + continue; + } + if holds_history(&events, script, entry, target_anchor.height) { + found.push(script.clone()); + } else { + unproductive += 1; + } + } + if found.is_empty() { + Ok(ShardRead::Unproductive { + count: unproductive, + }) + } else { + Ok(ShardRead::Done { matched: found }) + } + } + } +} + +/// The block a shard's coverage ends on for this sync: its own terminal +/// block, or the target when the shard runs past it. +fn endpoint_of(entry: &transparent_filter::ShardMapEntry, target_anchor: &Anchor) -> Anchor { + if entry.end_height > target_anchor.height { + target_anchor.clone() + } else { + Anchor { + height: entry.end_height, + hash: entry.terminal_block_hash.clone(), + } + } +} + +/// Which of `scripts` a shard's filter matches, by index. +fn match_filter( + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + genesis: BlockHash, + scripts: &[Vec], + bytes: &[u8], +) -> Result, SyncError> { + let profile = transparent_filter::range_profile(&map.profile) + .ok_or_else(|| SyncError::UnknownProfile(map.profile.clone()))?; + let validated = + transparent_filter::validate_range_filter(bytes, FilterLimits::default(), profile)?; + let terminal = BlockHash::from_display_hex(&entry.terminal_block_hash)?; + let key = ShardKey::derive( + &map.profile, + genesis, + entry.shard_id, + entry.start_height, + entry.end_height, + terminal, + ); + let owned: Vec = scripts + .iter() + .map(|script| ScriptBytes::new(script.clone())) + .collect(); + Ok(transparent_filter::match_range_scripts( + &validated, key, &owned, + )?) +} + +/// Runs `work` against a shard, verifying the manifest first and lifting the +/// service's two refusals: a withdrawn revision propagates for the map +/// refresh. An overload returns incomplete so the host can schedule a later pass. +#[allow(clippy::too_many_arguments)] +async fn with_refusals( + store: &mut S, + map: &ShardMap, + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + geometry: &ServiceGeometry, + clients: &mut Clients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + mut work: F, +) -> Result, SyncError> +where + F: AsyncFnMut( + &mut S, + &mut GeometryClients, + &mut T, + &mut ByteCharges, + &ShardManifest, + ) -> Result, SyncError>, +{ + let _ = limits; + let mut manifest: Option = None; + loop { + let verified = match &manifest { + Some(verified) => verified, + None => match fetch_manifest(entry, previous_digest, map, transport, charges).await { + Ok(verified) => { + if verified.schema != geometry.schema { + return Err(SyncError::ManifestMismatch { + shard_id: entry.shard_id, + field: "schema", + }); + } + manifest.insert(verified) + } + Err(SyncError::Client(ClientError::Stale(stale))) => { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes: 0, + }) + } + Err(SyncError::Client(ClientError::Overloaded(_overloaded))) => { + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::Overloaded { + shard_id: entry.shard_id, + }, + pending: store.pending()?.len(), + })); + } + Err(other) => return Err(other), + }, + }; + let prepared = clients.prepare(&verified.geometry, &geometry.geometries)?; + match work(store, prepared, transport, charges, verified).await { + Ok(stopped) => return Ok(stopped), + Err(SyncError::Client(ClientError::Stale(stale))) => { + return Err(SyncError::StaleRevision { + shard_id: entry.shard_id, + stale, + refreshes: 0, + }) + } + Err(SyncError::Client(ClientError::Overloaded(_overloaded))) => { + // Nothing partial is lost: every commit so far stands, and + // the work resumes from the store on the next attempt. + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::Overloaded { + shard_id: entry.shard_id, + }, + pending: store.pending()?.len(), + })); + } + Err(SyncError::Client(ClientError::SetupMismatch { + revision, + table, + segment, + })) => { + // The setup held for that segment is not the one the service + // answers under. It is forgotten, here and in the store, so + // the next attempt fetches it again; this read still fails, + // as any failed request does, and counts against no entry. + let prepared = clients.prepare(&verified.geometry, &geometry.geometries)?; + match table { + Table::Directory => prepared.directory.forget(&revision, table, segment), + Table::Pages(_) => prepared.pages.forget(&revision, table, segment), + } + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + store.drop_setup(&SetupKey { + set_digest, + revision_digest: revision.clone(), + table, + segment, + })?; + return Err(SyncError::Client(ClientError::SetupMismatch { + revision, + table, + segment, + })); + } + Err(other) => return Err(other), + } + } +} + +/// Whether a budget has been reached. +fn budget_stopped(charges: &ByteCharges, limits: &WorkLimits) -> Option { + if let Some(max) = limits.max_queries { + if charges.queries() >= max { + return Some(IncompleteReason::QueryBudget); + } + } + if let Some(max) = limits.max_private_bytes { + let private = charges.setup_bytes() + charges.query_upload() + charges.query_download(); + if private >= max { + return Some(IncompleteReason::ByteBudget); + } + } + None +} + +/// Fetches a matched shard's manifest and verifies it against the map. +async fn fetch_manifest( + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + map: &ShardMap, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, +) -> Result { + let (raw, cost) = transport + .manifest(entry.shard_id, &entry.manifest_digest) + .await + .map_err(|error| SyncError::from(classify_transport(error)))?; + charges.add_manifest(cost); + verify_manifest(&raw, entry, previous_digest, map) +} + +/// Checks a manifest against the map entry that named it, the entry before +/// it, and this build's registry. +/// +/// The digest first: the bytes must hash to the digest the map names, or +/// nothing else in them is worth reading. Then every field a retrieval relies +/// on. The parent link ties the manifest to the map's previous entry, so a +/// manifest that is individually well formed but belongs to a different +/// sequence of shards is refused too. Segment shapes are checked against the +/// registry entry the manifest names, which is what makes the geometry a +/// property of the verified manifest rather than of the map alone. +/// +/// None of this is an independence proof. The map, the manifest and the +/// tables come from one publisher; agreement among them rules out corruption, +/// stale data and mixed publications, not omission. +fn verify_manifest( + raw: &[u8], + entry: &transparent_filter::ShardMapEntry, + previous_digest: &str, + map: &ShardMap, +) -> Result { + let shard_id = entry.shard_id; + let mismatch = |field: &'static str| SyncError::ManifestMismatch { shard_id, field }; + let manifest: ShardManifest = serde_json::from_slice(raw) + .map_err(|error| SyncError::Invalid(format!("shard {shard_id} manifest: {error}")))?; + if manifest.directory_segments.len() > 64 + || manifest.page_children.len() > 32 + || manifest + .page_children + .iter() + .any(|c| c.page_segments.len() > 64) + { + return Err(mismatch("resource_bounds")); + } + if manifest.digest() != entry.manifest_digest { + return Err(mismatch("digest")); + } + if !transparent_shard::manifest::supported_schema(&manifest.schema) { + return Err(SyncError::Schema { + served: manifest.schema.clone(), + expected: transparent_shard::SCHEMA, + }); + } + if manifest.layout != transparent_shard::ManifestLayout::current() { + return Err(mismatch("layout")); + } + if manifest.tag_salt_counter > transparent_shard::tag::MAX_TAG_SALT_COUNTER { + return Err(mismatch("tag_salt_counter")); + } + if manifest.network != map.network { + return Err(mismatch("network")); + } + if manifest.genesis_hash != map.genesis_hash { + return Err(mismatch("genesis_hash")); + } + if manifest.profile != map.profile { + return Err(mismatch("profile")); + } + if manifest.shard_id != entry.shard_id { + return Err(mismatch("shard_id")); + } + if manifest.geometry != entry.geometry { + return Err(mismatch("geometry")); + } + if manifest.start_height != entry.start_height || manifest.end_height != entry.end_height { + return Err(mismatch("range")); + } + if manifest.parent_block_hash != entry.parent_block_hash { + return Err(mismatch("parent_block_hash")); + } + if manifest.terminal_block_hash != entry.terminal_block_hash { + return Err(mismatch("terminal_block_hash")); + } + if manifest.filter_hash != entry.filter_hash { + return Err(mismatch("filter_hash")); + } + if manifest.sealed != entry.sealed { + return Err(mismatch("sealed")); + } + if manifest.revision != entry.revision { + return Err(mismatch("revision")); + } + if manifest.parent_manifest_digest != previous_digest { + return Err(mismatch("parent_manifest_digest")); + } + // The map's `page_segments` counts the directory shard's page shards. + if manifest.directory_segments.len() as u32 != entry.directory_segments + || manifest.page_children.len() as u32 != entry.page_segments + { + return Err(mismatch("segment_count")); + } + let geometry = transparent_shard::layout::by_name(&manifest.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(manifest.geometry.clone()))?; + for segment in &manifest.directory_segments { + if segment.rows != geometry.directory_rows + || segment.row_bytes != geometry.directory_row_bytes as u32 + { + return Err(mismatch("directory_segment_shape")); + } + } + // Every page shard: contiguous, in order, sealed but for an open last one + // of a tail, ending on the shard's terminal, segments of the registry's + // shape. A walk relies on each of these. + manifest + .check_children(geometry.page_rows, geometry.page_row_bytes as u32) + .map_err(|_| mismatch("page_children"))?; + for child in &manifest.page_children { + BlockHash::from_display_hex(&child.terminal_block_hash) + .map_err(|_| mismatch("page_children"))?; + } + if let Some(seal) = map.seal.get(&manifest.geometry) { + if manifest.seal.scripts_target != seal.max_scripts + || manifest.seal.page_rows_target != seal.max_page_rows + { + return Err(mismatch("seal")); + } + } + directory_choice(&manifest)?; + Ok(manifest) +} + +/// The manifest's directory choice table, if it publishes one. +/// +/// A table that does not decode, or that indexes a different number of +/// scripts than the shard places, stops the sync. Falling back to two queries +/// would be safe for this shard, but it would hide a malformed publication +/// behind a working one. +fn directory_choice( + manifest: &ShardManifest, +) -> Result, SyncError> { + manifest.directory_choice().map_err(|error| { + SyncError::Invalid(format!("shard {} manifest: {error}", manifest.shard_id)) + }) +} + +/// How long to wait before re-asking an overloaded service. +/// +/// The service's own figure is preferred, because it knows why it refused, but +/// it is capped: a delay a wallet cannot sanity-check is a delay a +/// misconfigured or hostile service could use to park a sync inside one call. +/// Either way the wait is jittered by up to half again, so wallets refused +/// together do not return together. + +/// Refetches the map after a withdrawn revision, and checks it continues the +/// one in hand. +/// +/// Returns the map and the digest of the bytes it came from, so a later refusal +/// naming that same digest can be recognised as unresolvable rather than +/// spending another refresh on it. +async fn refresh_map( + started_from: &ShardMap, + covered_through: u64, + filters: &mut impl FilterSource, + charges: &mut ByteCharges, +) -> Result<(ShardMap, String), SyncError> { + let (bytes, cost) = filters + .shard_map() + .await + .map_err(|error| SyncError::Transport(crate::transport::describe_error(error.as_ref())))?; + charges.map_bytes += cost; + let fresh: ShardMap = serde_json::from_slice(&bytes).map_err(|error| { + SyncError::Invalid(format!("refreshed shard map is malformed: {error}")) + })?; + fresh.check_shape().map_err(|error| { + SyncError::Invalid(format!("refreshed shard map is malformed: {error}")) + })?; + check_continuation(started_from, &fresh, covered_through).map_err(SyncError::MapDiverged)?; + let digest = hex::encode(::digest(&bytes)); + Ok((fresh, digest)) +} + +/// Checks a refreshed map is the same shard set, continued. +/// +/// A map fetched mid-sync is not independently trusted. The caller validated +/// the map this sync started from against the wallet's accepted chain, and a +/// refresh inherits that validation only over the range the two agree on. So +/// the set's identity must be unchanged, and every shard whose range is already +/// covered must be the *same entry* — sealed content is immutable, so a +/// disagreement there means a different set rather than a longer one. +/// +/// Everything this function rules out is what makes the recovery sound: once it +/// passes, the shard the sync resumes at begins exactly where coverage stopped, +/// so no range can be skipped and none can need retracting. +fn check_continuation( + started_from: &ShardMap, + fresh: &ShardMap, + covered_through: u64, +) -> Result<(), String> { + if fresh.genesis_hash != started_from.genesis_hash { + return Err("it names a different genesis block".into()); + } + if fresh.network != started_from.network { + return Err("it names a different network".into()); + } + if fresh.profile != started_from.profile { + return Err("it names a different filter profile".into()); + } + if fresh.range_envelope_version != started_from.range_envelope_version { + return Err("it names a different range envelope version".into()); + } + if fresh.start_height != started_from.start_height { + return Err("its coverage begins at a different height".into()); + } + // A re-cut redraws the boundaries this walk was planned over and gives + // shard ids to other ranges, even where it lies above everything covered + // so far. This sync ends; the next starts from the re-cut map, whose + // declaration lets it keep what the store holds. + if fresh.recut_epoch() != started_from.recut_epoch() { + return Err(format!( + "it is re-cut epoch {}, this sync started from epoch {}", + fresh.recut_epoch(), + started_from.recut_epoch() + )); + } + // New geometries may appear as a set grows into another tier; the seal + // parameters of a geometry already in use may not change, because they are + // what the boundaries were derived from. + for (name, seal) in &started_from.seal { + if let Some(refreshed) = fresh.seal.get(name) { + if refreshed != seal { + return Err(format!("it reseals geometry {name}")); + } + } + } + for (index, before) in started_from.shards.iter().enumerate() { + if before.end_height > covered_through { + break; + } + match fresh.shards.get(index) { + Some(after) if after == before => {} + Some(_) => { + return Err(format!( + "shard {} covers height {} but is not the shard this sync read", + before.shard_id, before.end_height + )) + } + None => { + return Err(format!( + "it withdraws shard {}, whose range this sync already covered", + before.shard_id + )) + } + } + } + // A map that ends below what is already covered withdraws history the + // wallet has read, which is the same disagreement as a re-cut shard. + if let Some(last) = fresh.shards.last() { + if last.end_height < covered_through { + return Err(format!( + "its coverage ends at {}, below the {covered_through} this sync already read", + last.end_height + )); + } + } else if covered_through >= started_from.start_height { + return Err("it publishes no shards at all".into()); + } + Ok(()) +} + +/// Checks the shard the walk resumes at after a refresh. +/// +/// The walk resumes by shard id, which is sound only while the id still names +/// a range starting where the withdrawn one did. `check_continuation` already +/// refuses a refreshed map that changes covered history or its re-cut epoch, +/// so with a well-formed map this holds; it is checked again here because +/// resuming anywhere else would skip or repeat heights. +fn check_resume( + resumed: &transparent_filter::ShardMapEntry, + withdrawn: &transparent_filter::ShardMapEntry, +) -> Result<(), String> { + if resumed.start_height != withdrawn.start_height { + return Err(format!( + "it moves shard {} from height {} to {}", + withdrawn.shard_id, withdrawn.start_height, resumed.start_height + )); + } + Ok(()) +} + +/// Salt the wallet derives from a verified manifest. Records are accepted +/// only under this salt. +fn tag_salt_of(manifest: &ShardManifest) -> Result<[u8; 32], SyncError> { + let terminal = BlockHash::from_display_hex(&manifest.terminal_block_hash).map_err(|error| { + SyncError::Invalid(format!( + "shard {} terminal block hash: {error}", + manifest.shard_id + )) + })?; + Ok(transparent_shard::tag_salt( + manifest.shard_id, + &terminal, + manifest.tag_salt_counter, + )) +} + +/// The directory phase for one shard: the candidate rows for every matched +/// script, inline events committed, chain walks recorded; then the walks. +/// +/// A script whose replies fail on content fails its entry alone: it gets no +/// coverage and no walk, as an entry with inline history or a false-positive +/// match sends no pages either, and the shard's one commit still lands for +/// every other script, counting the failure. Every directory query planned for +/// a script is sent before any of its replies is decoded, so the queries sent +/// are the same whatever came back, and in the order a lookahead stashed them. +/// A retry after an overload sends them again for every script without a +/// saved walk, failed ones included, so it too is the honest retry. +#[allow(clippy::too_many_arguments)] +async fn retrieve_shard_into( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + matched: &[Vec], + unmatched: &[Vec], + choice: Option<&transparent_shard::ChoiceTable>, + salt: &[u8; 32], + clients: &mut GeometryClients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result, SyncError> { + let shard_id = entry.shard_id; + let geometry = transparent_shard::layout::by_name(&entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(entry.geometry.clone()))?; + let revision = entry.manifest_digest.as_str(); + // An overload can occur after the directory commit but before the walks + // finish. with_refusals calls this function again on retry. Resume those + // durable walks first: rediscovering their directories would enqueue + // duplicate work and leave the original walks pending after the retry. + let pending: Vec<_> = store + .pending()? + .into_iter() + .filter(|p| p.shard_id == shard_id && p.revision_digest == revision) + .collect(); + let mut resumed = Vec::with_capacity(pending.len()); + for item in &pending { + if !held(store, &failures.entries, shard_id, &item.script)? || item.dummy_debt.is_some() { + resumed.push(item.clone()); + } + } + if !resumed.is_empty() { + if let Some(stopped) = finish_pages( + store, + entry, + manifest, + &resumed, + clients, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await? + { + return Ok(Some(stopped)); + } + } + // A retry after an overload asks again for every matched script without + // a saved walk, as an honest retry does for inline entries and false + // positives. An entry that already failed in this sync is asked again + // too, so the queries do not show which entries failed, but its replies + // are not read: it is not counted twice, covered, or walked. + let mut unowed = Vec::with_capacity(matched.len()); + for script in matched { + if !pending.iter().any(|p| &p.script == script) { + let failed = held(store, &failures.entries, shard_id, script)?; + unowed.push((script, failed)); + } + } + let matched = unowed; + if matched.is_empty() { + // The earlier directory commit also covered all unmatched scripts. + return Ok(None); + } + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + open_table( + store, + &set_digest, + shard_id, + revision, + Table::Directory, + entry.directory_segments, + &mut clients.directory, + transport, + charges, + None, + ) + .await?; + + if let Some(reason) = budget_stopped(charges, limits) { + return Ok(Some(Completion::Incomplete { + reason, + pending: store.pending()?.len(), + })); + } + + let mut events: Vec = Vec::new(); + let mut covered: Vec> = unmatched.to_vec(); + let mut pending_upsert: Vec = Vec::new(); + let mut failed_here: Vec> = Vec::new(); + + for (script, failed) in matched { + // With a published choice table, exactly the one candidate row it + // names is queried. Without one, both candidate rows are. Either way + // the query count per matched script is fixed for the shard: querying + // the first and stopping on a hit would make it depend on where the + // script landed, which is a function of the script. Candidates are + // taken over the shard's whole logical row space; each names one row + // within a segment, and every segment answers it. + let wanted = transparent_shard::script_tag(salt, script); + let rows = geometry.directory_rows * entry.directory_segments as u64; + let candidates = candidate_rows(shard_id, script, rows); + let queried: &[u64] = match choice { + Some(table) => std::slice::from_ref(&candidates[table.choice(shard_id, script)]), + None => &candidates, + }; + let mut replies = Vec::with_capacity(queried.len()); + for &row in queried { + let (_, within) = transparent_shard::layout::split_row(row, geometry.directory_rows); + let directory_rows = clients.directory.rows(); + let answer = clients + .directory + .fetch_row( + transport, + shard_id, + revision, + Table::Directory, + entry.directory_segments, + directory_rows, + within as usize, + charges, + ) + .await; + replies.push((row, answer?)); + } + if failed { + continue; + } + let found = match entry_in(entry, manifest, geometry, &wanted, &replies) { + Ok(found) => found, + Err(SyncError::Content(_)) => { + failed_here.push(script.clone()); + continue; + } + Err(other) => return Err(other), + }; + match found { + // A filter match with no directory entry is either a false positive + // or a script outside the private tables' coverage. Both are + // wasted work rather than errors; coverage still advances. + None => covered.push(script.clone()), + Some(found) => { + for event in &found.inline { + events.push(StoredEvent { + script: script.clone(), + event: *event, + shard_id, + revision_digest: revision.to_string(), + }); + } + match found.head() { + Some((child, row)) => pending_upsert.push(PendingPages { + id: None, + shard_id, + revision_digest: revision.to_string(), + script: script.clone(), + cursor: RunCursor { + child, + row, + older_runs: None, + }, + fragments: Vec::new(), + carried: found.inline.clone(), + dummy_debt: None, + attempts: 0, + target_anchor: Some(target_anchor.clone()), + }), + None => covered.push(script.clone()), + } + } + } + } + + // The directory phase is one commit: inline events, coverage for every + // script that needs no pages, and the walk owed for the rest. The walks + // that follow resume from the store even if nothing else of this call + // survives. + let pending_ids_before: std::collections::BTreeSet = + store.pending()?.iter().filter_map(|p| p.id).collect(); + commit_bounded( + store, + target_anchor, + ShardCommit { + source_anchor: None, + shard_id, + revision_digest: revision.to_string(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events, + covered_scripts: covered, + pending_upsert, + pending_complete: Vec::new(), + cached_runs: Vec::new(), + failed_entries: failed_here.clone(), + }, + )?; + for script in &failed_here { + record_failure(store, &mut failures.entries, shard_id, script)?; + } + let owed: Vec = store + .pending()? + .into_iter() + .filter(|p| { + p.shard_id == shard_id + && p.revision_digest == revision + && p.id.is_some_and(|id| !pending_ids_before.contains(&id)) + }) + .collect(); + if owed.is_empty() { + return Ok(None); + } + finish_pages( + store, + entry, + manifest, + &owed, + clients, + transport, + charges, + limits, + target_anchor, + chain, + failures, + ) + .await +} + +/// The entry tagged `wanted` among the replies to one script's candidate +/// rows, if there is one, checked as far as it can be before a page is read: +/// every reply decodes, the tag is in at most one row, the inline events lie +/// in the shard, and the run it points to is a row a page query can select. +/// Any of these failing is a content failure of the entry. +fn entry_in( + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + geometry: &transparent_shard::layout::Geometry, + wanted: &[u8], + replies: &[(u64, Reply)], +) -> Result, SyncError> { + let shard_id = entry.shard_id; + let children = manifest.page_children.len() as u32; + let mut found: Option<(u64, DirectoryEntry)> = None; + for (row, reply) in replies { + for raw in reply { + let decoded = transparent_shard::records::decode_directory_row(raw, children) + .map_err(|error| SyncError::Content(format!("shard {shard_id}: {error}")))?; + for candidate in decoded { + // The row is selected by a hash of the raw script. The tag + // derived from this manifest's salt is what settles a hit. + // A tag from any other salt does not match. + if candidate.tag[..] != *wanted { + continue; + } + match &found { + // The two candidate hashes can name the same row. The row + // is still fetched twice, so the query count does not + // depend on the script, and the second sighting is the + // same entry, not a second. + Some((seen_row, _)) if seen_row == row => {} + Some(_) => { + return Err(SyncError::Content(format!( + "shard {shard_id} holds a script tag twice" + ))); + } + None => found = Some((*row, candidate)), + } + } + } + } + let Some((_, found)) = found else { + return Ok(None); + }; + if found.inline.iter().any(|event| { + let height = event_height(event); + height < entry.start_height || height > entry.end_height + }) { + return Err(SyncError::Content(format!( + "shard {shard_id} holds an inline event outside its range" + ))); + } + if let Some((child, row)) = found.head() { + locate(manifest, geometry, shard_id, child, u64::from(row))?; + } + Ok(Some(found)) +} + +/// Walks the chains still owed for one directory shard revision, privately. +/// +/// Every query of `pages-` selects among that page shard's first +/// [`PageChild::query_rows`]. See [`walk_chains`] for the walk itself. +#[allow(clippy::too_many_arguments)] +async fn finish_pages( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + owed: &[PendingPages], + clients: &mut GeometryClients, + transport: &mut T, + charges: &mut ByteCharges, + limits: &WorkLimits, + target_anchor: &Anchor, + chain: &impl ChainView, + failures: &mut Failures, +) -> Result, SyncError> { + if entry.end_height < target_anchor.height { + match chain.is_accepted(entry.end_height, &entry.terminal_block_hash) { + Acceptance::Accepted => {} + Acceptance::Unknown => { + return Ok(Some(Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { + height: entry.end_height, + }, + pending: store.pending()?.len(), + })) + } + Acceptance::Rejected => { + return Err(SyncError::Invalid( + "pending shard endpoint rejected by wallet chain".into(), + )) + } + } + } + let mut rows = PirRows { + entry, + manifest, + client: &mut clients.pages, + transport, + charges, + limits, + }; + walk_chains( + store, + entry, + manifest, + owed, + &mut rows, + target_anchor, + failures, + ) + .await +} + +/// One page row a walk asks for: page shard `child`, and a row over its +/// logical row space, which is its segments concatenated. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct RowRequest { + child: u32, + row: u64, +} + +/// One private reply: the row from every table segment. +/// +/// A reply that does not verify as a whole (its length, its revision binding, +/// the parameter epoch it was made under) fails its request, not an entry: +/// the service produces those without knowing which row the query selects, +/// so they cannot be aimed at one. Only the rows' own content, decoded and +/// checked, can fail an entry. +type Reply = Vec>; + +/// Where a walk's page rows come from: privately, or in tests, directly. +trait RowSource { + /// Why the walk must stop before another round, if it must. + fn stopped(&self) -> Option; + /// Replies for a prefix of the requests, in order. A budget may stop + /// dispatch between requests; each reply contains every table segment. + async fn fetch( + &mut self, + store: &mut S, + requests: &[RowRequest], + ) -> Result, SyncError>; + /// A query whose reply is not read: row 0 of page shard `child`. Every + /// query to a table selects among the same rows, so it is the length of + /// any other query to that table, and the service cannot tell it apart. + fn dummy(&self, child: u32) -> RowRequest { + RowRequest { child, row: 0 } + } +} + +/// Page rows by private query against one directory shard revision. +struct PirRows<'a, T: ShardTransport> { + entry: &'a transparent_filter::ShardMapEntry, + manifest: &'a ShardManifest, + client: &'a mut TableClient, + transport: &'a mut T, + charges: &'a mut ByteCharges, + limits: &'a WorkLimits, +} + +impl RowSource for PirRows<'_, T> { + fn stopped(&self) -> Option { + budget_stopped(self.charges, self.limits) + } + + /// Sends a round's queries together when the transport overlaps + /// requests, one at a time otherwise. Which queries are sent is the + /// walk's alone; a batch changes only when. + async fn fetch( + &mut self, + store: &mut S, + requests: &[RowRequest], + ) -> Result, SyncError> { + let shard_id = self.entry.shard_id; + let revision = self.entry.manifest_digest.as_str(); + let geometry = transparent_shard::layout::by_name(&self.entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(self.entry.geometry.clone()))?; + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + if self.limits.max_private_bytes.is_none() && self.transport.concurrency() > 1 { + let remaining = self + .limits + .max_queries + .map(|limit| limit.saturating_sub(self.charges.queries()) as usize) + .unwrap_or(requests.len()); + let requests = &requests[..requests.len().min(remaining)]; + // Each request: its table, segment count, query rows and row within. + let mut planned = Vec::with_capacity(requests.len()); + for request in requests { + let child = self + .manifest + .child(request.child) + .ok_or_else(|| SyncError::Invalid("a walk left the page shards".into()))?; + let table = Table::Pages(request.child); + let segments = child.page_segments.len() as u32; + open_table( + store, + &set_digest, + shard_id, + revision, + table, + segments, + self.client, + self.transport, + self.charges, + None, + ) + .await?; + let (_, within) = + transparent_shard::layout::split_row(request.row, geometry.page_rows); + planned.push(( + table, + segments, + child.query_rows() as usize, + within as usize, + )); + } + let mut answers = Vec::with_capacity(planned.len()); + if self.transport.concurrency() <= 1 || planned.len() <= 1 { + for (table, segments, query_rows, within) in planned { + answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + query_rows, + within, + self.charges, + ) + .await?, + ); + } + return Ok(answers); + } + let mut queries = Vec::with_capacity(planned.len()); + for (table, _, query_rows, within) in &planned { + queries.push( + self.client + .prepare_prefix(revision, *table, *query_rows, *within)?, + ); + } + let batch: Vec> = planned + .iter() + .zip(&queries) + .map(|((table, ..), query)| ShardRequest::Query { + shard_id, + revision, + table: *table, + body: &query.body, + }) + .collect(); + let replies = self.transport.batch(&batch).await; + drop(batch); + for (((table, segments, query_rows, within), query), reply) in + planned.into_iter().zip(queries).zip(replies) + { + match reply { + Some(Ok((response, _))) => { + self.charges.add_query( + table, + query.body.len() as u64, + response.len() as u64, + ); + answers.push( + self.client + .decode(revision, table, segments, query, &response)?, + ); + } + Some(Err(error)) => return Err(SyncError::Client(classify_transport(error))), + None => answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + query_rows, + within, + self.charges, + ) + .await?, + ), + } + } + return Ok(answers); + } + let mut answers = Vec::with_capacity(requests.len()); + // A finite byte allowance is checked after each atomic response. Opening + // all setups or dispatching a whole batch first would exceed that contract. + for request in requests { + if self.stopped().is_some() { + break; + } + let child = self + .manifest + .child(request.child) + .ok_or_else(|| SyncError::Invalid("a walk left the page shards".into()))?; + let table = Table::Pages(request.child); + let segments = child.page_segments.len() as u32; + if !open_table( + store, + &set_digest, + shard_id, + revision, + table, + segments, + self.client, + self.transport, + self.charges, + Some(self.limits), + ) + .await? + { + break; + } + if self.stopped().is_some() { + break; + } + let (_, within) = transparent_shard::layout::split_row(request.row, geometry.page_rows); + answers.push( + self.client + .fetch_row( + self.transport, + shard_id, + revision, + table, + segments, + child.query_rows() as usize, + within as usize, + self.charges, + ) + .await?, + ); + } + Ok(answers) + } +} + +/// The history cached below a run, and the page shard of the cached run it +/// starts from. +type CachedBelow = (u8, Vec); + +/// One script's chain walk in progress. +struct Walk { + item: PendingPages, + /// The first fragment of the run at the cursor, once read. + head: Option, + /// Runs read and committed in this call, newest first, with their page + /// shard: what the run cache is filled from once the walk ends. + walked: Vec<(u8, Vec)>, + done: bool, + /// After a content failure: the page shard and number of queries the + /// walk had committed to and not sent, sent as dummies in the next round, + /// after which the walk ends. + dummies: Option<(u8, u32)>, +} + +/// A walk's failure, with the queries valid replies had already committed it +/// to: `dummies` of them, to page shard `child`. +struct Failure { + error: SyncError, + child: u8, + dummies: u32, +} + +impl From for Failure { + fn from(error: SyncError) -> Self { + Self { + error, + child: 0, + dummies: 0, + } + } +} + +/// Walks the chains of runs `owed` describes, all scripts' rounds together, +/// newest run first, committing each run once it is read and checked. +/// +/// A round reads each run's first fragment or a prefix of its remaining +/// fragments. Checked fragments are saved between passes, so a one-query +/// budget can advance even a long run. A complete run is committed before +/// checking the next request's budget. A walk reaching a cached sealed run +/// stops there and takes that history. +/// +/// Refused, as content failures: a link to the same or a later page shard; +/// an `older_runs` that does not drop by exactly one per hop, which is how a +/// chain cut short or extended shows; fragments out of ordinal, count or +/// greedy boundary; events not strictly increasing from the oldest run into +/// the inline events; and an event in a page shard other than the one open +/// when it was pushed out — the event two places after it must lie in that +/// page shard's range. +/// +/// A content failure fails its walk alone; the other walks finish. The failed +/// walk keeps its pending item, so a later sync retries it, and the failure is +/// counted against the entry. It still sends what valid replies had already +/// committed it to, the fragments of its current run not yet asked for, as +/// dummy queries to that run's page shard, and then nothing. A run whose link +/// names a page shard the manifest does not declare, or a row outside that +/// table, sends one dummy to its own page shard in the link's place, unless +/// the honest walk would not query a head after this run. Nothing that +/// depends on the bad reply is sent. +async fn walk_chains( + store: &mut S, + entry: &transparent_filter::ShardMapEntry, + manifest: &ShardManifest, + owed: &[PendingPages], + rows: &mut R, + target_anchor: &Anchor, + failures: &mut Failures, +) -> Result, SyncError> { + let geometry = transparent_shard::layout::by_name(&entry.geometry) + .ok_or_else(|| SyncError::UnknownGeometry(entry.geometry.clone()))?; + let identities: Vec = manifest.child_digests().iter().map(hex::encode).collect(); + let salts = manifest + .page_children + .iter() + .map(|child| { + let terminal = BlockHash::from_display_hex(&child.terminal_block_hash)?; + Ok(transparent_shard::page_tag_salt( + manifest.shard_id, + child.index, + child.end_height, + &terminal, + child.tag_salt_counter, + )) + }) + .collect::, SyncError>>()?; + let walker = Walker { + entry, + manifest, + geometry, + identities: &identities, + salts: &salts, + target_anchor, + }; + let mut walks: Vec = Vec::with_capacity(owed.len()); + for item in owed { + let mut walk = Walk { + item: item.clone(), + head: None, + walked: Vec::new(), + done: false, + dummies: None, + }; + if let Some((child, count)) = item.dummy_debt { + let page = walker.child(child)?; + let space = geometry.page_rows * page.page_segments.len() as u64; + if count == 0 || u64::from(count) > space.saturating_add(1) { + return Err(SyncError::Invalid("invalid pending dummy debt".into())); + } + walker.locate(child, 0)?; + walk.dummies = Some((child, count)); + } else if let Some(raw) = item.fragments.first() { + let head = walker + .fragment( + &item.script, + item.cursor.child, + std::slice::from_ref(raw), + 0, + None, + ) + .map_err(Failure::from) + .and_then(|head| { + walker.check_located( + store, + &item.script, + &head, + &item.cursor, + item.fragments.len(), + )?; + Ok(head) + }); + match head { + Ok(head) => walk.head = Some(head), + Err(failure) => walker.fail(store, &mut walk, failures, failure)?, + } + } + walks.push(walk); + } + loop { + // A walk at a run the store holds history for stops there. + for walk in walks + .iter_mut() + .filter(|w| !w.done && w.dummies.is_none() && w.head.is_none()) + { + let child = walk.item.cursor.child; + let stopped = match walker.cached(store, &walk.item.script, child) { + Ok(Some(older)) => walker.finish(store, walk, Vec::new(), Some((child, older))), + Ok(None) => Ok(()), + Err(error) => Err(error), + }; + if let Err(error) = stopped { + walker.fail(store, walk, failures, error.into())?; + } + } + if walks.iter().all(|walk| walk.done) { + return Ok(None); + } + if rows.stopped().is_some() { + for walk in walks.iter_mut().filter(|w| !w.done && w.dummies.is_none()) { + if walk + .head + .as_ref() + .is_some_and(|head| walk.item.fragments.len() == head.fragment_count as usize) + { + if let Err(failure) = walker.complete(store, walk) { + walker.fail(store, walk, failures, failure)?; + } + } + } + if walks.iter().all(|walk| walk.done) { + return Ok(None); + } + } + if let Some(reason) = rows.stopped() { + return Ok(Some(Completion::Incomplete { + reason, + pending: store.pending()?.len(), + })); + } + + // This round's rows, every walk's together. + let mut requests: Vec = Vec::new(); + let mut spans: Vec<(usize, usize, Option)> = Vec::new(); + for walk in walks.iter_mut() { + let start = requests.len(); + let mut older = None; + if walk.dummies.is_none() && !walk.done { + match walker.plan(store, walk) { + Ok((planned, below)) => { + requests.extend(planned); + older = below; + } + Err(error) => walker.fail(store, walk, failures, error.into())?, + } + } + if let Some((child, count)) = walk.dummies { + requests.extend((0..count).map(|_| rows.dummy(u32::from(child)))); + } + spans.push((start, requests.len(), older)); + } + let answers = rows.fetch(store, &requests).await?; + if answers.len() > requests.len() + || answers.is_empty() && !requests.is_empty() && rows.stopped().is_none() + { + return Err(SyncError::Invalid("a round lost an answer".into())); + } + for (walk, (start, end, older)) in walks.iter_mut().zip(spans) { + if walk.done { + continue; + } + // Only the successfully returned prefix pays debt. A budget can + // split this round, and a checkpoint/cold resume must retain the rest. + // If fetch refuses the round, no prefix is committed, exactly as for + // an honest fragment round whose local replies are discarded. + if let Some((child, count)) = walk.dummies { + let paid = answers.len().saturating_sub(start).min(end - start) as u32; + if paid > 0 { + let remaining = count.checked_sub(paid).ok_or_else(|| { + SyncError::Invalid("a dummy round returned extra answers".into()) + })?; + walk.item.dummy_debt = (remaining > 0).then_some((child, remaining)); + walk.item.target_anchor = Some(target_anchor.clone()); + walker.commit( + store, + &walk.item.script, + Vec::new(), + Some(walk.item.clone()), + None, + Vec::new(), + )?; + walk.dummies = walk.item.dummy_debt; + walk.done = remaining == 0; + } + continue; + } + if start >= answers.len() && start != end { + continue; + } + let available = if start == end { + &[][..] + } else { + &answers[start..end.min(answers.len())] + }; + if let Err(failure) = walker.advance(store, walk, available, older) { + walker.fail(store, walk, failures, failure)?; + } + } + } +} + +/// What a walk checks against, for one directory shard revision. +struct Walker<'a> { + entry: &'a transparent_filter::ShardMapEntry, + manifest: &'a ShardManifest, + geometry: &'static transparent_shard::layout::Geometry, + /// Each page shard's chained identity, in hex: the run cache's key. + identities: &'a [String], + /// Each page shard's tag salt. + salts: &'a [[u8; 32]], + target_anchor: &'a Anchor, +} + +/// Page shard `child` of a directory shard, which its manifest must declare. +fn page_child(manifest: &ShardManifest, shard_id: u64, child: u8) -> Result<&PageChild, SyncError> { + manifest.child(u32::from(child)).ok_or_else(|| { + SyncError::Content(format!( + "shard {shard_id} names page shard {child}, which it does not declare" + )) + }) +} + +/// The request for `row` of page shard `child`, if the row is one a query of +/// that table can select. A pointer is a claim like any other. +fn locate( + manifest: &ShardManifest, + geometry: &transparent_shard::layout::Geometry, + shard_id: u64, + child: u8, + row: u64, +) -> Result { + let page = page_child(manifest, shard_id, child)?; + if page.page_segments.is_empty() { + return Err(SyncError::Content(format!( + "shard {shard_id} points into page shard {child}, which holds no rows" + ))); + } + let space = geometry.page_rows * page.page_segments.len() as u64; + let (_, within) = transparent_shard::layout::split_row(row, geometry.page_rows); + if row >= space || within >= page.query_rows() { + return Err(SyncError::Content(format!( + "shard {shard_id} page {row} is outside page shard {child}'s table" + ))); + } + Ok(RowRequest { + child: u32::from(child), + row, + }) +} + +impl Walker<'_> { + fn child(&self, child: u8) -> Result<&PageChild, SyncError> { + page_child(self.manifest, self.entry.shard_id, child) + } + + /// The request for `row` of page shard `child`; see [`locate`]. + fn locate(&self, child: u8, row: u64) -> Result { + locate( + self.manifest, + self.geometry, + self.entry.shard_id, + child, + row, + ) + } + + /// Checks a run's first fragment, read at `cursor`, before the walk + /// relies on it: every fragment it declares must be a row a page query + /// can select, or the run is not located at all. Then its link: the + /// older-run count its chain promised (see [`check_head`]) and a run to + /// link to that starts at a row of an earlier, declared page shard. A + /// link that fails any of these still leaves this run located, so the + /// walk owes the queries for its fragments not yet held, of `held`, and + /// one in the link's place unless the honest walk would not follow it, + /// all as dummies: the round the honest walk would send. + fn check_located( + &self, + store: &S, + script: &[u8], + head: &PageEntry, + cursor: &RunCursor, + held: usize, + ) -> Result<(), Failure> { + let last = u64::from(cursor.row) + u64::from(head.fragment_count.max(1)) - 1; + self.locate(cursor.child, last)?; + let link = check_head(head, cursor).and_then(|()| match head.link.previous() { + Some((child, row)) => self.locate(child, u64::from(row)).map(|_| ()), + None => Ok(()), + }); + let Err(error) = link else { + return Ok(()); + }; + let link_query = self.owed_link_query(store, script, head, cursor)?; + Err(Failure { + error, + child: cursor.child, + dummies: head.fragment_count.saturating_sub(held.max(1) as u32) + link_query, + }) + } + + /// Whether the honest walk would query a head after the run at `cursor`. + /// + /// It would not when the chain already promised this run was the oldest, + /// nor when this script's previous run is cached: a catch-up stops there + /// and sends nothing for it. When the link still names an earlier, + /// declared page shard (only its row or its older-run count is bad), that + /// is the page shard the honest walk would look up in the cache. Otherwise + /// the previous run is taken to be in the page shard just below. Either + /// way a bad link must not add a query the honest walk omits. + fn owed_link_query( + &self, + store: &S, + script: &[u8], + head: &PageEntry, + cursor: &RunCursor, + ) -> Result { + if cursor.older_runs == Some(0) { + return Ok(0); + } + let previous = match head.link.previous() { + Some((child, _)) if child < cursor.child && self.child(child).is_ok() => child, + _ => match cursor.child.checked_sub(1) { + Some(below) => below, + None => return Ok(1), + }, + }; + Ok(u32::from(self.cached(store, script, previous)?.is_none())) + } + + /// One round's requests for a walk, and the history cached below its + /// run when the walk will stop there. + fn plan( + &self, + store: &S, + walk: &Walk, + ) -> Result<(Vec, Option), SyncError> { + let cursor = walk.item.cursor; + let Some(head) = &walk.head else { + return Ok(( + vec![self.locate(cursor.child, u64::from(cursor.row))?], + None, + )); + }; + let mut requests = Vec::new(); + for ordinal in walk.item.fragments.len() as u32..head.fragment_count { + let row = u64::from(cursor.row) + u64::from(ordinal); + requests.push(self.locate(cursor.child, row)?); + } + let mut older = None; + if let Some((child, row)) = head.link.previous() { + older = self + .cached(store, &walk.item.script, child)? + .map(|events| (child, events)); + if older.is_none() { + requests.push(self.locate(child, u64::from(row))?); + } + } + Ok((requests, older)) + } + + /// The history cached for `script` from its run in page shard `child` + /// back to the start of the directory shard, when that page shard is + /// sealed. Each cached run holds only its own events and names the cached + /// run below it, so the history is put back together by following those + /// names; a missing run anywhere along the way is a miss. + fn cached( + &self, + store: &S, + script: &[u8], + child: u8, + ) -> Result>, SyncError> { + if !self.child(child)?.sealed { + return Ok(None); + } + let corrupt = |why: &str| SyncError::Store(StoreError::Corrupt(why.into())); + let mut index = usize::from(child); + let mut runs = Vec::new(); + loop { + let Some(run) = store.cached_run(&self.identities[index], script)? else { + return Ok(None); + }; + if run.events.is_empty() { + return Err(corrupt("a cached run holds no events")); + } + runs.push(run.events); + let Some(older) = run.older else { break }; + // The identity of a page shard covers every older one, so the run + // below must be in an earlier page shard of this directory shard. + index = self.identities[..index] + .iter() + .position(|identity| *identity == older) + .ok_or_else(|| { + corrupt("a cached run names a run outside the page shards below it") + })?; + } + let mut history: Vec = Vec::new(); + for run in runs.into_iter().rev() { + if let (Some(last), Some(first)) = (history.last(), run.first()) { + if last.sort_key() >= first.sort_key() { + return Err(corrupt("cached runs are not in order")); + } + } + history.extend(run); + } + Ok(Some(history)) + } + + /// The one fragment of `script`'s run in page shard `child` with this + /// ordinal, among every segment's row. Exactly one may claim it. + fn fragment( + &self, + script: &[u8], + child: u8, + answers: &[Vec], + ordinal: u32, + count: Option, + ) -> Result { + let shard_id = self.entry.shard_id; + let wanted = transparent_shard::script_tag(&self.salts[usize::from(child)], script); + // A packed row also carries other scripts' histories. Those are + // discarded, and deliberately not used: two of a wallet's own scripts + // sharing a row still cost two fetches, so the request transcript does + // not depend on which scripts share a row. + let mut found = None; + for raw in answers { + let decoded = transparent_shard::page_row::decode_page_row(raw).map_err(|error| { + SyncError::Content(format!("shard {shard_id} page shard {child}: {error}")) + })?; + for candidate in decoded { + if candidate.tag != wanted || candidate.ordinal != ordinal { + continue; + } + if count.is_some_and(|count| candidate.fragment_count != count) { + return Err(SyncError::Content(format!( + "shard {shard_id} page shard {child}: fragment {ordinal} disagrees with \ + its run's fragment count" + ))); + } + if found.is_some() { + return Err(SyncError::Content(format!( + "shard {shard_id} page shard {child} holds fragment {ordinal} twice" + ))); + } + found = Some(candidate); + } + } + found.ok_or_else(|| { + SyncError::Content(format!( + "shard {shard_id} page shard {child}: the row a link located does not hold \ + fragment {ordinal} of the run" + )) + }) + } + + /// Takes one walk through one round's replies. + fn advance( + &self, + store: &mut S, + walk: &mut Walk, + answers: &[Reply], + older: Option, + ) -> Result<(), Failure> { + let cursor = walk.item.cursor; + let script = walk.item.script.clone(); + if walk.head.is_none() { + let [answer] = answers else { + return Err(SyncError::Invalid("a round lost an answer".into()).into()); + }; + let head = self.fragment(&script, cursor.child, answer, 0, None)?; + self.check_located(store, &script, &head, &cursor, 1)?; + walk.item.fragments.push(encode_fragment(&head)?); + walk.head = Some(head); + return Ok(self.commit( + store, + &script, + Vec::new(), + Some(walk.item.clone()), + None, + Vec::new(), + )?); + } + let head = walk.head.as_ref().expect("head decoded above").clone(); + let count = head.fragment_count as usize; + if walk.item.fragments.len() > count { + return Err(SyncError::Invalid("a run has too many fragments".into()).into()); + } + let remaining = count - walk.item.fragments.len(); + let (answers, next) = answers.split_at(remaining.min(answers.len())); + if next.len() > 1 { + return Err(SyncError::Invalid("a round has extra answers".into()).into()); + } + // The run's fragments this round asked for and a budget left unsent: + // what the walk still owes if this run fails. + let unsent = (remaining - answers.len()) as u32; + let failing = |error: SyncError| Failure { + error, + child: cursor.child, + dummies: unsent, + }; + let mut boundary = validate_page_boundary(None, &head.events).map_err(failing)?; + let mut run = head.events.clone(); + // Recheck saved boundaries before using them after a restart. + for (ordinal, raw) in walk.item.fragments.iter().enumerate().skip(1) { + let fragment = self + .fragment( + &script, + cursor.child, + std::slice::from_ref(raw), + ordinal as u32, + Some(head.fragment_count), + ) + .map_err(failing)?; + boundary = + validate_page_boundary(Some(&boundary), &fragment.events).map_err(failing)?; + run.extend(fragment.events); + } + for answer in answers { + let ordinal = walk.item.fragments.len() as u32; + let fragment = self + .fragment( + &script, + cursor.child, + answer, + ordinal, + Some(head.fragment_count), + ) + .map_err(failing)?; + boundary = + validate_page_boundary(Some(&boundary), &fragment.events).map_err(failing)?; + run.extend(fragment.events.iter().copied()); + walk.item.fragments.push(encode_fragment(&fragment)?); + } + if walk.item.fragments.len() < count { + return Ok(self.commit( + store, + &script, + Vec::new(), + Some(walk.item.clone()), + None, + Vec::new(), + )?); + } + check_run(self.child(cursor.child)?, &run, &walk.item.carried)?; + walk.walked.push((cursor.child, run.clone())); + match head.link.previous() { + None => Ok(self.finish(store, walk, run, None)?), + Some(_) if older.is_some() => Ok(self.finish(store, walk, run, older)?), + Some((child, row)) => { + let mut following = run.clone(); + following.extend(walk.item.carried.iter().copied()); + following.truncate(2); + let mut progressed = walk.item.clone(); + progressed.cursor = RunCursor { + child, + row, + older_runs: Some(head.link.older_runs - 1), + }; + progressed.carried = following; + progressed.fragments.clear(); + // A bad head for the next run fails the walk, but not the run + // already checked: that is committed, and the walk resumes at + // the next run's head. + let mut failure = None; + let mut next_head = None; + if let [answer] = next { + let checked = self + .fragment(&script, child, answer, 0, None) + .map_err(Failure::from) + .and_then(|head| { + self.check_located(store, &script, &head, &progressed.cursor, 1)?; + Ok(head) + }); + match checked { + Ok(head) => { + progressed.fragments.push(encode_fragment(&head)?); + next_head = Some(head); + } + Err(error) => failure = Some(error), + } + } + progressed.attempts += 1; + self.commit( + store, + &script, + run, + Some(progressed.clone()), + None, + Vec::new(), + ) + .map_err(|error| match (error, failure.take()) { + ( + SyncError::Store(StoreError::HistoryMetadataLimit), + Some( + known @ Failure { + error: SyncError::Content(_), + .. + }, + ), + ) => known, + (error, _) => error.into(), + })?; + walk.item = progressed; + walk.head = next_head; + failure.map_or(Ok(()), Err) + } + } + } + + /// Takes a walk whose run is read in full through to its end, with no + /// replies: what a stopped round still finishes. + fn complete(&self, store: &mut S, walk: &mut Walk) -> Result<(), Failure> { + let older = match walk.head.as_ref().and_then(|head| head.link.previous()) { + Some((child, _)) => self + .cached(store, &walk.item.script, child)? + .map(|events| (child, events)), + None => None, + }; + self.advance(store, walk, &[], older) + } + + /// Ends a walk after `failure`. A content failure is counted against the + /// walk's entry, which keeps its pending item for a later sync, and the + /// walk sends the dummies it owes in the next round, if any; any other + /// failure is the call's. + fn fail( + &self, + store: &mut S, + walk: &mut Walk, + failures: &mut Failures, + failure: Failure, + ) -> Result<(), SyncError> { + let Failure { + error, + child, + dummies, + } = failure; + let SyncError::Content(_) = error else { + return Err(error); + }; + let entry = self.entry; + // A failed round must not save its uncommitted prefix: an honest + // refused round retries that prefix too. Attach debt to the last + // durable pending item, atomically with the failure count. + let id = walk + .item + .id + .ok_or_else(|| SyncError::Invalid("uncommitted failed walk".into()))?; + let mut durable = store + .pending()? + .into_iter() + .find(|held| { + held.id == Some(id) + && held.shard_id == entry.shard_id + && held.revision_digest == entry.manifest_digest + && held.script == walk.item.script + }) + .ok_or_else(|| SyncError::Invalid("missing failed walk".into()))?; + durable.dummy_debt = (dummies > 0).then_some((child, dummies)); + walk.item = durable; + commit_bounded( + store, + self.target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + failed_entries: vec![walk.item.script.clone()], + pending_upsert: vec![walk.item.clone()], + ..Default::default() + }, + )?; + record_failure( + store, + &mut failures.entries, + entry.shard_id, + &walk.item.script, + )?; + if dummies > 0 { + walk.dummies = Some((child, dummies)); + } else { + walk.done = true; + } + Ok(()) + } + + /// Ends a walk: `run` is the last run read, `older` the page shard and + /// history cached below it, if the walk stopped at a cached run. Fills the + /// run cache for every run this call read in a sealed page shard: each + /// with its own events and the name of the cached run below it. + fn finish( + &self, + store: &mut S, + walk: &mut Walk, + run: Vec, + older: Option, + ) -> Result<(), SyncError> { + let (mut below, older) = match older { + Some((child, events)) => (Some(self.identities[usize::from(child)].clone()), events), + None => (None, Vec::new()), + }; + let newer = run.first().or(walk.item.carried.first()); + if let (Some(last), Some(first)) = (older.last(), newer) { + if last.sort_key() >= first.sort_key() { + return Err(SyncError::Content( + "cached history is not older than the run that links to it".into(), + )); + } + } + let mut cached_runs = Vec::new(); + for (child, events) in walk.walked.iter().rev() { + // Only the newest run can be in the open page shard, so every run + // below a cached one is cached too. + if !self.child(*child)?.sealed { + break; + } + let identity = self.identities[usize::from(*child)].clone(); + cached_runs.push(CachedRun { + child_digest: identity.clone(), + script: walk.item.script.clone(), + shard_id: self.entry.shard_id, + events: events.clone(), + older: below.replace(identity), + }); + } + let mut events = older; + events.extend(run); + let script = walk.item.script.clone(); + self.commit(store, &script, events, None, walk.item.id, cached_runs)?; + walk.done = true; + Ok(()) + } + + /// Commits a step of one walk: the events it read, and either the walk's + /// next state or its end, with the script's coverage. + fn commit( + &self, + store: &mut S, + script: &[u8], + events: Vec, + progressed: Option, + finished: Option, + cached_runs: Vec, + ) -> Result<(), SyncError> { + let entry = self.entry; + let done = progressed.is_none(); + commit_bounded( + store, + self.target_anchor, + ShardCommit { + source_anchor: None, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + sealed: entry.sealed, + start_height: entry.start_height, + end_height: entry.end_height, + terminal_block_hash: entry.terminal_block_hash.clone(), + events: events + .into_iter() + .map(|event| StoredEvent { + script: script.to_vec(), + event, + shard_id: entry.shard_id, + revision_digest: entry.manifest_digest.clone(), + }) + .collect(), + covered_scripts: if done { + vec![script.to_vec()] + } else { + Vec::new() + }, + pending_upsert: progressed.into_iter().collect(), + pending_complete: finished.into_iter().collect(), + cached_runs, + failed_entries: Vec::new(), + }, + )?; + Ok(()) + } +} + +/// Checks a run's first fragment against the cursor that located it. +/// +/// Its `older_runs` must be the one fewer the newer run promised — which is +/// what makes a chain cut short, or one that carries on past where it should +/// end, a refusal rather than a shorter or longer history. It may not claim +/// more older runs than there are earlier page shards, and its link must +/// point to an earlier page shard: never this one, never a later one. +fn check_head(head: &PageEntry, cursor: &RunCursor) -> Result<(), SyncError> { + let link = head.link; + match cursor.older_runs { + Some(expected) if link.older_runs != expected => { + return Err(SyncError::Content(format!( + "a run in page shard {} reports {} older runs where its chain promised {expected}", + cursor.child, link.older_runs + ))) + } + _ if link.older_runs > cursor.child => { + return Err(SyncError::Content(format!( + "a run in page shard {} reports {} older runs", + cursor.child, link.older_runs + ))) + } + _ => {} + } + if let Some((child, _)) = link.previous() { + if child >= cursor.child { + return Err(SyncError::Content(format!( + "a run in page shard {} links to page shard {child}, which is not earlier", + cursor.child + ))); + } + } + Ok(()) +} + +/// Checks a run against the two events after it and its page shard. +/// +/// Events strictly increase from the run into `carried`. Each of the run's +/// events was pushed out when the event two places after it arrived, into the +/// page shard open then, so that event's height lies in this page shard's +/// range. A run placed in any other page shard fails here. +fn check_run( + child: &PageChild, + run: &[TransparentEvent], + carried: &[TransparentEvent], +) -> Result<(), SyncError> { + if run.is_empty() || carried.len() != transparent_shard::INLINE_EVENTS as usize { + return Err(SyncError::Content(format!( + "a run in page shard {} cannot be checked", + child.index + ))); + } + let following: Vec<&TransparentEvent> = run.iter().chain(carried).collect(); + if following + .windows(2) + .any(|pair| pair[0].sort_key() >= pair[1].sort_key()) + { + return Err(SyncError::Content(format!( + "a run in page shard {} is not older than the history after it", + child.index + ))); + } + for (index, _) in run.iter().enumerate() { + let height = event_height(following[index + 2]); + if height < child.start_height || height > child.end_height { + return Err(SyncError::Content(format!( + "an event in page shard {} was pushed out at height {height}, when that page \ + shard was not open", + child.index + ))); + } + } + Ok(()) +} + +fn validate_page_boundary( + previous: Option<&crate::store::PageBoundary>, + events: &[transparent_events::TransparentEvent], +) -> Result { + let first = events + .first() + .ok_or_else(|| SyncError::Content("empty fragment".into()))?; + if let Some(previous) = previous { + if previous.last_event.sort_key() > first.sort_key() + || previous.event_bytes as usize + + transparent_shard::compact::event_len(first, Some(&previous.last_event)) + <= transparent_shard::packing::FRAGMENT_PAYLOAD + { + return Err(SyncError::Content("noncanonical fragment boundary".into())); + } + } + Ok(crate::store::PageBoundary { + event_bytes: transparent_shard::compact::encoded_len(events) as u32, + last_event: *events.last().expect("nonempty fragment"), + }) +} + +/// Fetches the published setup for every segment of a shard's table, once, +/// reusing what the store holds under the exact set, revision and segment. +/// +/// Each segment publishes its own `c1`, so each is opened separately; a shard +/// with one segment — the ordinary case — costs exactly one setup per table. +#[allow(clippy::too_many_arguments)] +async fn open_table( + store: &mut S, + set_digest: &str, + shard_id: u64, + revision: &str, + table: Table, + segments: u32, + client: &mut TableClient, + transport: &mut impl ShardTransport, + charges: &mut ByteCharges, + limits: Option<&WorkLimits>, +) -> Result { + if segments == 0 { + return Err(SyncError::Invalid(format!( + "shard {shard_id} declares no {} segments", + table.name() + ))); + } + for segment in 0..segments { + if client.is_open(revision, table, segment) { + continue; + } + let key = SetupKey { + set_digest: set_digest.to_string(), + revision_digest: revision.to_string(), + table, + segment, + }; + let (params, digest) = match store.setup(&key)? { + Some(blob) => (blob.public_params_base64, blob.public_params_sha256), + None => { + if limits + .and_then(|limits| budget_stopped(charges, limits)) + .is_some() + { + return Ok(false); + } + let (raw, cost) = transport + .setup(shard_id, revision, table, segment) + .await + .map_err(|error| SyncError::from(classify_transport(error)))?; + charges.add_setup(table, cost); + let parsed: serde_json::Value = serde_json::from_slice(&raw) + .map_err(|error| SyncError::Transport(error.to_string()))?; + let params = parsed["public_params"] + .as_str() + .ok_or_else(|| SyncError::Invalid("setup has no public_params".into()))? + .to_string(); + let digest = parsed["public_params_sha256"] + .as_str() + .ok_or_else(|| SyncError::Invalid("setup has no digest".into()))? + .to_string(); + (params, digest) + } + }; + client.open_segment(revision, table, segment, ¶ms, &digest)?; + // Kept only after the client verified it against its digest. + store.put_setup( + &key, + &SetupBlob { + public_params_base64: params, + public_params_sha256: digest, + }, + )?; + } + Ok(true) +} + +fn check_target_hash(map: &ShardMap, target: &Anchor) -> Result<(), SyncError> { + if map + .shard_for_height(target.height) + .is_some_and(|s| s.end_height == target.height && s.terminal_block_hash != target.hash) + { + return Err(SyncError::Invalid( + "publication endpoint disagrees with wallet-accepted target hash".into(), + )); + } + Ok(()) +} + +/// Commits only the wallet-accepted prefix while retaining the publication identity. +fn commit_bounded( + store: &mut S, + target: &Anchor, + mut commit: ShardCommit, +) -> Result { + let source = Anchor { + height: commit.end_height, + hash: commit.terminal_block_hash.clone(), + }; + for event in commit + .events + .iter() + .map(|e| &e.event) + .chain(commit.pending_upsert.iter().flat_map(|p| p.carried.iter())) + { + let height = event_height(event); + if height < commit.start_height || height > source.height { + return Err(SyncError::Content( + "event lies outside its published shard".into(), + )); + } + } + commit.source_anchor = Some(source); + if commit.end_height >= target.height { + commit.end_height = target.height; + commit.terminal_block_hash = target.hash.clone(); + } + // Events above the target are not kept. A walk's carried events are not + // history but what it checks the next run against, so they stay whole. + commit + .events + .retain(|e| event_height(&e.event) <= target.height); + // What a reply carried contradicting what the store holds is the reply's + // content failure, not the store's. + store.commit_shard(commit).map_err(|error| match error { + StoreError::Contradiction(why) => SyncError::Content(format!("contradiction: {why}")), + other => SyncError::Store(other), + }) +} + +/// A checked fragment, saved as a page row holding only it. +fn encode_fragment(fragment: &PageEntry) -> Result, SyncError> { + Ok(transparent_shard::page_row::encode_page_row( + std::slice::from_ref(fragment), + )?) +} + +/// A sync's content failures so far: the entries, and the dummy queries +/// failed walks still owe. A shard read retried after an overload sends what +/// is still owed before the walk ends. +#[derive(Default)] +struct Failures { + entries: Vec, +} + +/// Whether `script`'s entry in shard `shard_id` is held out of this sync: +/// it failed on content earlier in the sync, or has failed +/// [`MAX_CONTENT_FAILURES`] times. A held entry's replies are not read; it +/// is queried again only by a shard read retried after an overload, which +/// asks what the first attempt asked. +fn held( + store: &S, + failed: &[FailedEntry], + shard_id: u64, + script: &[u8], +) -> Result { + Ok(failed + .iter() + .any(|entry| entry.shard_id == shard_id && entry.script == script) + || store.entry_failures(shard_id, script)? >= MAX_CONTENT_FAILURES) +} + +/// [`held`], listing an entry held for its failure count in the report the +/// first time this sync meets it. +fn hold( + store: &S, + failed: &mut Vec, + shard_id: u64, + script: &[u8], +) -> Result { + if failed + .iter() + .any(|entry| entry.shard_id == shard_id && entry.script == script) + { + return Ok(true); + } + let failures = store.entry_failures(shard_id, script)?; + if failures < MAX_CONTENT_FAILURES { + return Ok(false); + } + failed.push(FailedEntry { + shard_id, + script: script.to_vec(), + failures, + }); + Ok(true) +} + +/// Lists an entry whose failure a commit just counted, with the store's count. +fn record_failure( + store: &S, + failed: &mut Vec, + shard_id: u64, + script: &[u8], +) -> Result<(), SyncError> { + let failures = store.entry_failures(shard_id, script)?; + match failed + .iter_mut() + .find(|entry| entry.shard_id == shard_id && entry.script == script) + { + Some(entry) => entry.failures = failures, + None => failed.push(FailedEntry { + shard_id, + script: script.to_vec(), + failures, + }), + } + Ok(()) +} + +pub(crate) fn event_height(event: &transparent_events::TransparentEvent) -> u64 { + match event { + transparent_events::TransparentEvent::Receive(e) => u64::from(e.height), + transparent_events::TransparentEvent::Spend(e) => u64::from(e.height), + } +} + +fn accepted_at(chain: &impl ChainView, map: &ShardMap, height: u64) -> Result { + let hash = chain.hash_at(height).or_else(|| { + map.shards.iter().find_map(|s| { + if s.end_height == height { + Some(s.terminal_block_hash.clone()) + } else if s.start_height.checked_sub(1) == Some(height) { + Some(s.parent_block_hash.clone()) + } else { + None + } + }) + }); + match hash { + Some(hash) if chain.is_accepted(height, &hash) == Acceptance::Accepted => { + Ok(Anchor { height, hash }) + } + _ => Err(SyncError::Invalid(format!( + "wallet has no accepted rollback hash at {height}" + ))), + } +} + +/// The report of a sync stopped because the wallet's chain cannot confirm a +/// block at `height`: whatever was already rolled back, and nothing read. +#[allow(clippy::too_many_arguments)] +fn chain_unknown( + store: &S, + map: &ShardMap, + target_anchor: &Anchor, + charges: ByteCharges, + height: u64, + rolled_back_to: Option, + replaced_revisions: Vec, + first_commit: u64, +) -> Result { + let (covered, settled) = coverage_summary(store, map, target_anchor.height)?; + Ok(SyncReport { + ledger: store.ledger()?, + charges, + matched_shards: Vec::new(), + unproductive_matches: 0, + covered_through: covered, + settled_through: settled, + provisional: provisional_of(store, map)?, + map_refreshes: 0, + completion: Completion::Incomplete { + reason: IncompleteReason::ChainUnknown { height }, + pending: store.pending()?.len(), + }, + rolled_back_to, + replaced_revisions, + scripts_added: 0, + commits: store.last_commit()? - first_commit, + failed_entries: Vec::new(), + }) +} + +fn incomplete_at( + store: &S, + reason: IncompleteReason, +) -> Result { + Ok(SyncReport { + ledger: store.ledger()?, + charges: ByteCharges::default(), + matched_shards: vec![], + unproductive_matches: 0, + covered_through: 0, + settled_through: 0, + provisional: vec![], + map_refreshes: 0, + completion: Completion::Incomplete { + reason, + pending: store.pending()?.len(), + }, + rolled_back_to: None, + replaced_revisions: vec![], + scripts_added: 0, + commits: 0, + failed_entries: Vec::new(), + }) +} + +#[cfg(test)] +mod cap_tests; diff --git a/crates/z-transparent-pir-v2/src/sync/cap_tests.rs b/crates/z-transparent-pir-v2/src/sync/cap_tests.rs new file mode 100644 index 0000000..01df064 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/sync/cap_tests.rs @@ -0,0 +1,351 @@ +use super::*; +use crate::store::{ScriptEntry, ScriptOrigin}; +use crate::{ + shared_store::{SharedStore, MAX_STORE_BYTES}, + types, Limits, Pass, Status, Step, +}; +use bincode::Options; +use transparent_events::{ReceiveEvent, Txid}; +use transparent_shard::page_row::RunLink; +#[allow(dead_code)] +#[path = "../../tests/fixture.rs"] +mod fixture; + +struct Fixture { + _dir: tempfile::TempDir, + entry: transparent_filter::ShardMapEntry, + manifest: ShardManifest, + identities: Vec, + salts: Vec<[u8; 32]>, + target: Anchor, + script: Vec, + head: PageEntry, + fragments: Vec>, + bad: PageEntry, + carried: Vec, +} +fn receive(height: u32, tx: u8, output: u32) -> TransparentEvent { + TransparentEvent::Receive(ReceiveEvent { + metadata: None, + height, + txid: Txid([tx; 32]), + transaction_index: 0, + output_index: output, + value: 1, + coinbase: false, + }) +} +impl Fixture { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let mut script = vec![0x76, 0xa9, 0x14]; + script.extend([1; 20]); + script.extend([0x88, 0xac]); + let mut run = vec![receive(2, 2, 0), receive(3, 3, 0)]; + run.extend((0..78).map(|output| receive(60, 40, output))); + let carried = vec![receive(61, 61, 0), receive(62, 62, 0)]; + let events = std::iter::once(receive(1, 1, 0)) + .chain(run) + .chain(carried.iter().copied()) + .map(|event| (transparent_filter::ScriptBytes::new(script.clone()), event)) + .collect(); + let mut spec = fixture::ShardSpec::first(0, 100, true, events); + spec.page_span = 50; + let map = fixture::publish( + dir.path(), + &[spec], + fixture::SEAL, + fixture::synthetic, + vec![], + ); + let entry = map.shards[0].clone(); + let published = dir.path().join(&entry.manifest_digest); + let manifest: ShardManifest = + serde_json::from_slice(&std::fs::read(published.join("manifest.json")).unwrap()) + .unwrap(); + let row = |child| { + let bytes = std::fs::read(published.join(format!("pages-{child}.0.bin"))).unwrap(); + transparent_shard::page_row::decode_page_row(&bytes[..fixture::GEOMETRY.page_row_bytes]) + .unwrap() + .remove(0) + }; + let head = row(1); + assert_eq!(head.fragment_count, 2); + let bytes = std::fs::read(published.join("pages-1.0.bin")).unwrap(); + let fragments: Vec<_> = bytes + .chunks_exact(fixture::GEOMETRY.page_row_bytes) + .take(head.fragment_count as usize) + .map(|row| row.to_vec()) + .collect(); + let count: usize = fragments + .iter() + .map(|row| { + transparent_shard::page_row::decode_page_row(row).unwrap()[0] + .events + .len() + }) + .sum(); + assert_eq!(count, 80); + let mut bad = row(0); + bad.fragment_count = 3; + bad.link = RunLink { + prev_child: 0, + prev_row: 1, + older_runs: 1, + }; + let identities = manifest.child_digests().iter().map(hex::encode).collect(); + let salts = manifest + .page_children + .iter() + .map(|child| { + transparent_shard::page_tag_salt( + manifest.shard_id, + child.index, + child.end_height, + &BlockHash::from_display_hex(&child.terminal_block_hash).unwrap(), + child.tag_salt_counter, + ) + }) + .collect(); + let target = Anchor { + height: 100, + hash: fixture::synthetic(100).to_display_hex(), + }; + Self { + _dir: dir, + entry, + manifest, + identities, + salts, + target, + script, + head, + fragments, + bad, + carried, + } + } + fn walker(&self) -> Walker<'_> { + Walker { + entry: &self.entry, + manifest: &self.manifest, + geometry: fixture::GEOMETRY, + identities: &self.identities, + salts: &self.salts, + target_anchor: &self.target, + } + } + fn scope(&self) -> types::Scope { + types::Scope { + source_id: "1".repeat(64), + scope_id: "2".repeat(64), + generation: 1, + anchor: types::Anchor { + height: 100, + hash: self.target.hash.clone(), + }, + scripts: vec![types::Script { + script: hex::encode(&self.script), + required_from: 0, + }], + accepted_headers: vec![ + types::Anchor { + height: 0, + hash: transparent_filter::MAINNET_GENESIS_DISPLAY.into(), + }, + types::Anchor { + height: 100, + hash: self.target.hash.clone(), + }, + ], + } + } + fn pass(&self, mixed: bool, event_cap: bool) -> (Pass, SharedStore, Walk, types::Checkpoint) { + let mut pass = Pass::new(self.scope(), None, Limits::default()).unwrap(); + let mut store = pass.fixture_store(); + store + .add_scripts(&[ScriptEntry { + script: self.script.clone(), + origin: ScriptOrigin::Derived, + required_from: 0, + }]) + .unwrap(); + let pending = PendingPages { + id: None, + shard_id: 0, + revision_digest: self.entry.manifest_digest.clone(), + script: self.script.clone(), + cursor: RunCursor { + child: u8::from(mixed), + row: 0, + older_runs: Some(u8::from(mixed)), + }, + carried: if mixed { + self.carried.clone() + } else { + self.head.events[..2].to_vec() + }, + fragments: if mixed { + self.fragments.clone() + } else { + vec![] + }, + dummy_debt: None, + attempts: 0, + target_anchor: Some(self.target.clone()), + }; + let events = if event_cap { + (0..65_536) + .map(|output| StoredEvent { + script: self.script.clone(), + event: receive(1, 90, output), + shard_id: 0, + revision_digest: self.entry.manifest_digest.clone(), + }) + .collect() + } else { + vec![] + }; + store + .commit_shard(ShardCommit { + events, + pending_upsert: vec![pending], + ..Default::default() + }) + .unwrap(); + if !event_cap { + let keys: Vec<_> = (0..8).map(|n| format!("{n:064x}")).collect(); + for key in &keys { + store + .inner + .borrow_mut() + .put_filter(key, &"c".repeat(64), false, &[]) + .unwrap(); + } + let mut remaining = MAX_STORE_BYTES - encoded(&store).len(); + for key in &keys { + let n = remaining.min(8 * 1024 * 1024); + remaining -= n; + store + .inner + .borrow_mut() + .put_filter(key, &"c".repeat(64), false, &vec![0; n]) + .unwrap(); + } + assert_eq!(remaining, 0); + assert_eq!(encoded(&store).len(), MAX_STORE_BYTES); + } + pass.finish_budget().unwrap(); + let Step::Checkpoint { checkpoint } = pass.advance().unwrap() else { + panic!("baseline checkpoint"); + }; + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + let item = store.pending().unwrap().remove(0); + let walk = Walk { + item, + head: mixed.then(|| self.head.clone()), + walked: vec![], + done: false, + dummies: None, + }; + (pass, store, walk, checkpoint) + } + fn assert_terminal( + &self, + pass: &mut Pass, + store: &SharedStore, + prior: &types::Checkpoint, + expected_cursor: RunCursor, + ) { + assert!(store.policy_limit.get()); + let pending = store.pending().unwrap(); + assert_eq!(pending[0].dummy_debt, Some((0, 2))); + assert_eq!(pending[0].cursor, expected_cursor); + let Step::Checkpoint { checkpoint } = pass.advance().unwrap() else { + panic!("new debt must checkpoint"); + }; + assert!(!checkpoint.complete); + assert_eq!(checkpoint.records, prior.records); + assert_eq!( + checkpoint.previous_checkpoint_id, + Some(prior.checkpoint_id.clone()) + ); + pass.acknowledge(&checkpoint.checkpoint_id).unwrap(); + assert!( + matches!(pass.advance().unwrap(),Step::Stopped {status:Status::Incomplete {reason}} if reason=="history_metadata_limit") + ); + let mut cold = Pass::new(self.scope(), Some(checkpoint), Limits::default()).unwrap(); + assert!( + matches!(cold.advance().unwrap(),Step::Stopped {status:Status::Incomplete {reason}} if reason=="history_metadata_limit") + ); + assert_eq!(cold.fixture_store().pending().unwrap()[0], pending[0]); + } +} +fn encoded(store: &SharedStore) -> Vec { + bincode::DefaultOptions::new() + .with_fixint_encoding() + .serialize(&*store.inner.borrow()) + .unwrap() +} + +#[test] +fn real_walker_failure_preserves_stamped_anchor_debt_at_byte_cap() { + let fixture = Fixture::new(); + let (mut pass, mut store, mut walk, prior) = fixture.pass(false, false); + let cursor = walk.item.cursor; + let failure = fixture + .walker() + .advance( + &mut store, + &mut walk, + &[vec![encode_fragment(&fixture.bad).unwrap()]], + None, + ) + .unwrap_err(); + assert!(matches!(failure.error, SyncError::Content(_))); + assert_eq!(failure.dummies, 2); + fixture + .walker() + .fail(&mut store, &mut walk, &mut Failures::default(), failure) + .expect("real stamped failure must preserve new debt"); + fixture.assert_terminal(&mut pass, &store, &prior, cursor); +} +fn mixed_prefix(event_cap: bool) { + let fixture = Fixture::new(); + let (mut pass, mut store, mut walk, prior) = fixture.pass(true, event_cap); + let cursor = walk.item.cursor; + let before = encoded(&store); + let failure = fixture + .walker() + .advance( + &mut store, + &mut walk, + &[vec![encode_fragment(&fixture.bad).unwrap()]], + None, + ) + .unwrap_err(); + assert!( + store.policy_limit.get(), + "checked run must hit the real local cap" + ); + assert!( + matches!(failure.error, SyncError::Content(_)), + "growth refusal must retain the known bad-next-head failure" + ); + assert_eq!(failure.dummies, 2); + assert_eq!(walk.item.cursor, cursor); + assert_eq!(encoded(&store), before); + fixture + .walker() + .fail(&mut store, &mut walk, &mut Failures::default(), failure) + .unwrap(); + fixture.assert_terminal(&mut pass, &store, &prior, cursor); +} +#[test] +fn real_walker_event_cap_retains_bad_next_head_debt() { + mixed_prefix(true) +} +#[test] +fn real_walker_byte_cap_retains_bad_next_head_debt() { + mixed_prefix(false) +} diff --git a/crates/z-transparent-pir-v2/src/sync_ahead.rs b/crates/z-transparent-pir-v2/src/sync_ahead.rs new file mode 100644 index 0000000..2c21ae0 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/sync_ahead.rs @@ -0,0 +1,497 @@ +// Adapted from valargroup/wallet-pir c9262e8431b4caa7cf2e2c55edbf0084c1f2ddb2. +// Copyright (c) 2025-2026 Roman Akhtariev. MIT; see LICENSE-MIT and UPSTREAM.md. +//! Cross-shard request concurrency: the requests of a pass, sent together. +//! +//! The sequential walk reads one shard at a time: its filter, its manifest, its +//! directory setup, its directory queries, its pages. Each request waits for +//! the one before, so a restore spanning several matched shards pays one round +//! trip per request. Most of those requests do not depend on each other. Once +//! every uncached filter is local, which shards match and which scripts matched +//! them is known before any private request, and so are the manifests and +//! directory setups the walk will ask for. Once the manifests are verified, so +//! are the directory rows. +//! +//! [`look_ahead`] computes exactly those requests, hands them to the transport +//! as two batches, and stashes the outcome of every one that was sent where the +//! walk will look for it: manifests and setups in [`Staged`], filters in +//! [`StagedFilters`], queries in each directory +//! [`TableClient`](crate::client::TableClient) with the secret that decodes +//! them. The walk then runs unchanged. A stashed outcome is consumed at most +//! once, at the point where the sequential walk would have made that request, +//! and a refusal or failure is handled there exactly as if the request had just +//! been made: a withdrawn revision refreshes the map, an overload backs off and +//! retries within the same attempt budget, anything else fails the sync with +//! the store as the sequential walk would have left it. An unsent request is +//! made by the walk in its place. The walk is still what verifies, commits, +//! retries and refreshes, so its semantics are the sequential walk's. +//! +//! # What stays the same +//! +//! - Which requests are made, per shard and per table: the lookahead derives +//! them from the same filter bytes, script lists, choice table and setup +//! cache the walk would use, and the walk consumes rather than repeats them. +//! No query is skipped because another response mentioned a script. +//! - Charges: a stashed reply is charged when the walk consumes it, with what +//! the transport reported, so totals equal the sequential walk's. +//! - Commits: still per shard, in shard order, by the walk. +//! +//! # What changes +//! +//! When requests are sent. The service sees a pass's manifests and setups +//! together, then its directory queries together, instead of interleaved with +//! commits. Page work is not looked ahead: which pages a script needs is known +//! only after its directory entry decodes. The chain walk batches each of its +//! own rounds instead, across every script it walks. +//! +//! Requests the walk ends up not using are extra on the wire. If the walk stops +//! or restarts before a shard it looked ahead to — an exhausted overload +//! budget, a withdrawn revision and the refresh after it, a hard error — the +//! requests already sent for later shards were seen by the service and never +//! used. Those shards matched, so the next sync asks the same of them; a +//! refresh re-walks the rest of the pass in sequence. Likewise, requests +//! already in flight when a batch meets its first failure complete, and those +//! for the same shard after a refused query go unused, because the walk +//! retries that shard from its first query. +//! +//! The lookahead stops at the first shard whose endpoint, filter or manifest +//! does not check out, which is where the walk would stop too. A budgeted sync +//! never looks ahead, because whether a later shard's queries are sent at all +//! depends on page work only the walk discovers. + +use super::{ + directory_choice, endpoint_of, held, match_filter, previous_digest_of, verify_manifest, + Clients, FailedEntry, ServiceGeometry, SyncError, +}; +use crate::adapters::{Acceptance, ChainView}; +use crate::client::Table; +use crate::store::{Anchor, SetupKey, WalletStore}; +use crate::transport::{BoxError, FilterSource, ShardReply, ShardRequest, ShardTransport}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use transparent_filter::{BlockHash, ShardMap, ShardMapEntry}; +use transparent_shard::build::candidate_rows; +use transparent_shard::manifest::ShardManifest; + +/// The outcome of a request made ahead of the walk: its bytes and what they +/// cost, or its refusal or failure. +type Answer = Result<(Vec, u64), BoxError>; + +/// A shard transport with replies already in hand. +pub(super) struct Staged<'a, T: ShardTransport> { + inner: &'a mut T, + manifests: HashMap<(u64, String), Answer>, + setups: HashMap<(u64, String, Table, u32), Answer>, +} + +impl<'a, T: ShardTransport> Staged<'a, T> { + pub(super) fn new(inner: &'a mut T) -> Self { + Self { + inner, + manifests: HashMap::new(), + setups: HashMap::new(), + } + } + + pub(super) fn clear(&mut self) { + self.manifests.clear(); + self.setups.clear(); + } +} + +impl ShardTransport for Staged<'_, T> { + async fn init(&mut self) -> Result<(Vec, u64), BoxError> { + self.inner.init().await + } + + async fn manifest( + &mut self, + shard_id: u64, + revision: &str, + ) -> Result<(Vec, u64), BoxError> { + match self.manifests.remove(&(shard_id, revision.to_string())) { + Some(reply) => reply, + None => self.inner.manifest(shard_id, revision).await, + } + } + + async fn setup( + &mut self, + shard_id: u64, + revision: &str, + table: Table, + segment: u32, + ) -> Result<(Vec, u64), BoxError> { + match self + .setups + .remove(&(shard_id, revision.to_string(), table, segment)) + { + Some(reply) => reply, + None => self.inner.setup(shard_id, revision, table, segment).await, + } + } + + async fn query( + &mut self, + shard_id: u64, + revision: &str, + table: Table, + body: &[u8], + ) -> Result, BoxError> { + self.inner.query(shard_id, revision, table, body).await + } + + fn concurrency(&self) -> usize { + self.inner.concurrency() + } + + async fn batch(&mut self, requests: &[ShardRequest<'_>]) -> Vec> { + self.inner.batch(requests).await + } +} + +/// A filter source with filters already in hand. +/// +/// Keyed by shard id alone, as [`FilterSource::filter`] is, so the stash is +/// cleared whenever the walk refreshes the map: a filter fetched for a +/// withdrawn revision must not be handed out for the one that replaced it. +pub(super) struct StagedFilters<'a, F: FilterSource> { + inner: &'a mut F, + filters: HashMap, +} + +impl<'a, F: FilterSource> StagedFilters<'a, F> { + pub(super) fn new(inner: &'a mut F) -> Self { + Self { + inner, + filters: HashMap::new(), + } + } + + pub(super) fn clear(&mut self) { + self.filters.clear(); + } +} + +impl FilterSource for StagedFilters<'_, F> { + fn uses_parents(&self) -> bool { + self.inner.uses_parents() + } + + async fn shard_map(&mut self) -> Result<(Vec, u64), BoxError> { + self.inner.shard_map().await + } + + async fn filter(&mut self, shard_id: u64) -> Result<(Vec, u64), BoxError> { + match self.filters.remove(&shard_id) { + Some(reply) => reply, + None => self.inner.filter(shard_id).await, + } + } + + fn prefetch(&mut self, shard_ids: &[u64]) { + self.inner.prefetch(shard_ids) + } + + async fn prepare_parents( + &mut self, + map: &ShardMap, + uncached: &[u64], + store: &mut dyn WalletStore, + ) -> Result { + self.inner.prepare_parents(map, uncached, store).await + } + + async fn parent_negative( + &mut self, + map: &ShardMap, + shard_id: u64, + scripts: &[Vec], + store: &mut dyn WalletStore, + ) -> Result<(bool, u64), BoxError> { + self.inner + .parent_negative(map, shard_id, scripts, store) + .await + } +} + +/// Drops everything stashed, so the walk requests afresh from here on. +pub(super) fn clear( + transport: &mut Staged<'_, T>, + filters: &mut StagedFilters<'_, F>, + clients: &mut Clients, +) { + transport.clear(); + filters.clear(); + for prepared in clients.prepared.values_mut() { + prepared.directory.clear_stash(); + } +} + +/// A shard the walk will query, and the scripts it will query it for. +struct Planned<'m> { + entry: &'m ShardMapEntry, + matched: Vec>, +} + +/// Sends the independent requests of one pass ahead of the walk. +/// +/// `order` is the walk's shard order and `work` its scripts per shard. Only a +/// store failure is an error: anything else ends the lookahead early and +/// leaves the rest to the walk, which meets the same condition in its own +/// place and handles it as it always has. +#[allow(clippy::too_many_arguments)] +pub(super) async fn look_ahead( + store: &S, + map: &ShardMap, + genesis: BlockHash, + order: &[usize], + work: &BTreeMap>>, + geometry: &ServiceGeometry, + clients: &mut Clients, + filters: &mut StagedFilters<'_, F>, + transport: &mut Staged<'_, T>, + target_anchor: &Anchor, + chain: &impl ChainView, + failed: &[FailedEntry], +) -> Result<(), SyncError> { + // Match every shard the walk will reach, as it would. Filters not in the + // store are taken from the source here and stashed for the walk, which + // still checks, caches and charges them. + let pending = store.pending()?; + let mut planned: Vec> = Vec::new(); + for &index in order { + let Some(entry) = map.shards.get(index) else { + break; + }; + let endpoint = endpoint_of(entry, target_anchor); + if chain.is_accepted(endpoint.height, &endpoint.hash) != Acceptance::Accepted { + break; + } + let scripts = work.get(&index).map(Vec::as_slice).unwrap_or_default(); + let bytes = match store.filter(&entry.manifest_digest, &entry.filter_hash)? { + Some(bytes) => bytes, + None => { + // Whatever came back is the walk's outcome for this fetch, a + // failure or bytes that do not hash to the map included: its + // own check of them is what starts a refresh. + let reply = filters.inner.filter(entry.shard_id).await; + let usable = match &reply { + Ok((bytes, _)) + if transparent_filter::filter_hash(bytes).to_display_hex() + == entry.filter_hash => + { + Some(bytes.clone()) + } + _ => None, + }; + filters.filters.insert(entry.shard_id, reply); + match usable { + Some(bytes) => bytes, + None => break, + } + } + }; + let Ok(matches) = match_filter(map, entry, genesis, scripts, &bytes) else { + break; + }; + if matches.is_empty() { + continue; + } + if transparent_shard::layout::by_name(&entry.geometry).is_none() { + break; + } + // Durable page work for this revision changes what the walk asks for + // first; leave the shard to it. + if pending + .iter() + .any(|p| p.shard_id == entry.shard_id && p.revision_digest == entry.manifest_digest) + { + continue; + } + // Entries the walk holds out of this sync are not queried. + let mut matched = Vec::with_capacity(matches.len()); + for &i in &matches { + if !held(store, failed, entry.shard_id, &scripts[i])? { + matched.push(scripts[i].clone()); + } + } + if matched.is_empty() { + continue; + } + planned.push(Planned { entry, matched }); + } + if planned.is_empty() { + return Ok(()); + } + + // First batch: every manifest, and every directory setup the walk would + // fetch because neither the client nor the store holds it. + let set_digest = store + .set_identity()? + .map(|identity| identity.digest()) + .unwrap_or_default(); + let mut requests: Vec> = Vec::new(); + for shard in &planned { + let entry = shard.entry; + let revision = entry.manifest_digest.as_str(); + requests.push(ShardRequest::Manifest { + shard_id: entry.shard_id, + revision, + }); + for segment in 0..entry.directory_segments { + let open = clients + .prepared + .get(&entry.geometry) + .is_some_and(|prepared| { + prepared + .directory + .is_open(revision, Table::Directory, segment) + }); + let key = SetupKey { + set_digest: set_digest.clone(), + revision_digest: revision.to_string(), + table: Table::Directory, + segment, + }; + if !open && store.setup(&key)?.is_none() { + requests.push(ShardRequest::Setup { + shard_id: entry.shard_id, + revision, + table: Table::Directory, + segment, + }); + } + } + } + let replies = transport.inner.batch(&requests).await; + let mut manifests: HashMap = HashMap::new(); + for (request, reply) in requests.iter().zip(replies) { + // A sent request's outcome is kept whatever it was: a refusal reaches + // the walk where the sequential walk would have met it, and is + // handled there. An unsent one is simply made by the walk. + let Some(reply) = reply else { + continue; + }; + match *request { + ShardRequest::Manifest { shard_id, revision } => { + if let (Ok((raw, _)), Some(shard)) = ( + &reply, + planned.iter().find(|p| p.entry.shard_id == shard_id), + ) { + let previous = previous_digest_of(map, shard_id); + if let Ok(manifest) = verify_manifest(raw, shard.entry, &previous, map) { + manifests.insert(shard_id, manifest); + } + } + transport + .manifests + .insert((shard_id, revision.to_string()), reply); + } + ShardRequest::Setup { + shard_id, + revision, + table, + segment, + } => { + transport + .setups + .insert((shard_id, revision.to_string(), table, segment), reply); + } + ShardRequest::Query { .. } => unreachable!("the first batch holds no queries"), + } + } + + // Second batch: the directory queries, in walk order, prepared by the + // clients that will decode them. Rows are chosen exactly as the walk + // chooses them: the published choice when there is a table, both + // candidates when there is not. + struct Ahead<'m> { + entry: &'m ShardMapEntry, + geometry: String, + row: usize, + query: crate::client::PreparedQuery, + } + let mut queries: Vec> = Vec::new(); + 'shards: for shard in &planned { + let entry = shard.entry; + let Some(manifest) = manifests.get(&entry.shard_id) else { + break; + }; + let Ok(choice) = directory_choice(manifest) else { + break; + }; + let Ok(prepared) = clients.prepare(&manifest.geometry, &geometry.geometries) else { + break; + }; + let Some(layout) = transparent_shard::layout::by_name(&entry.geometry) else { + break; + }; + let rows = layout.directory_rows * u64::from(entry.directory_segments); + for script in &shard.matched { + let candidates = candidate_rows(entry.shard_id, script, rows); + let queried: &[u64] = match &choice { + Some(table) => { + std::slice::from_ref(&candidates[table.choice(entry.shard_id, script)]) + } + None => &candidates, + }; + for &row in queried { + let (_, within) = transparent_shard::layout::split_row(row, layout.directory_rows); + let Ok(query) = prepared.directory.prepare( + &entry.manifest_digest, + Table::Directory, + within as usize, + ) else { + break 'shards; + }; + queries.push(Ahead { + entry, + geometry: manifest.geometry.clone(), + row: within as usize, + query, + }); + } + } + } + if queries.is_empty() { + return Ok(()); + } + let requests: Vec> = queries + .iter() + .map(|ahead| ShardRequest::Query { + shard_id: ahead.entry.shard_id, + revision: &ahead.entry.manifest_digest, + table: Table::Directory, + body: &ahead.query.body, + }) + .collect(); + let replies = transport.inner.batch(&requests).await; + drop(requests); + // A revision's stash must be a prefix of what the walk will ask for it. + // After an unsent query the walk prepares and sends that one and the rest + // itself; after a refused or failed one it starts the shard over, so + // nothing later of that revision would be used either. + let mut ended: HashSet<&str> = HashSet::new(); + for (ahead, reply) in queries.into_iter().zip(replies) { + let revision = ahead.entry.manifest_digest.as_str(); + if ended.contains(revision) { + continue; + } + let Some(reply) = reply else { + ended.insert(revision); + continue; + }; + if reply.is_err() { + ended.insert(revision); + } + if let Some(prepared) = clients.prepared.get_mut(&ahead.geometry) { + prepared.directory.stash( + revision, + Table::Directory, + ahead.row, + ahead.query, + reply.map(|(bytes, _)| bytes), + ); + } + } + Ok(()) +} diff --git a/crates/z-transparent-pir-v2/src/transport.rs b/crates/z-transparent-pir-v2/src/transport.rs new file mode 100644 index 0000000..1627e77 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/transport.rs @@ -0,0 +1,87 @@ +use crate::client::Table; +use crate::store::WalletStore; +pub use transparent_wallet::transport::{ + BoxError, ByteCharges, Overloaded, ShardReply, ShardRequest, StaleRevision, +}; +pub(crate) fn describe_error(e: &(dyn std::error::Error + 'static)) -> String { + e.to_string() +} +pub trait FilterSource { + fn uses_parents(&self) -> bool { + false + } + async fn shard_map(&mut self) -> Result<(Vec, u64), BoxError>; + async fn filter(&mut self, id: u64) -> Result<(Vec, u64), BoxError>; + fn prefetch(&mut self, _ids: &[u64]) {} + async fn prepare_parents( + &mut self, + _map: &transparent_filter::ShardMap, + _ids: &[u64], + _s: &mut dyn WalletStore, + ) -> Result { + Ok(0) + } + async fn parent_negative( + &mut self, + _map: &transparent_filter::ShardMap, + _id: u64, + _scripts: &[Vec], + _s: &mut dyn WalletStore, + ) -> Result<(bool, u64), BoxError> { + Ok((false, 0)) + } +} +pub trait ShardTransport { + async fn init(&mut self) -> Result<(Vec, u64), BoxError>; + async fn manifest(&mut self, id: u64, rev: &str) -> Result<(Vec, u64), BoxError>; + async fn setup( + &mut self, + id: u64, + rev: &str, + table: Table, + segment: u32, + ) -> Result<(Vec, u64), BoxError>; + async fn query( + &mut self, + id: u64, + rev: &str, + table: Table, + body: &[u8], + ) -> Result, BoxError>; + fn concurrency(&self) -> usize { + 1 + } + async fn batch(&mut self, requests: &[ShardRequest<'_>]) -> Vec> { + let mut out = Vec::new(); + let mut failed = false; + for r in requests { + if failed { + out.push(None); + continue; + } + let result = match *r { + ShardRequest::Manifest { shard_id, revision } => { + self.manifest(shard_id, revision).await + } + ShardRequest::Setup { + shard_id, + revision, + table, + segment, + } => self.setup(shard_id, revision, table, segment).await, + ShardRequest::Query { + shard_id, + revision, + table, + body, + } => self.query(shard_id, revision, table, body).await.map(|b| { + let n = b.len() as u64; + (b, n) + }), + }; + failed = result.is_err(); + out.push(Some(result)); + } + out + } +} diff --git a/crates/z-transparent-pir-v2/src/types.rs b/crates/z-transparent-pir-v2/src/types.rs new file mode 100644 index 0000000..1db4746 --- /dev/null +++ b/crates/z-transparent-pir-v2/src/types.rs @@ -0,0 +1,155 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Table { + Directory, + Pages { child: u32 }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Operation { + Init, + Map, + Filter { + shard_id: u64, + }, + Manifest { + shard_id: u64, + revision: String, + }, + Setup { + shard_id: u64, + revision: String, + table: Table, + segment: u32, + }, + Query { + shard_id: u64, + revision: String, + table: Table, + body: String, + }, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Reply { + Data { body: String }, + Stale { map_digest_hint: Option }, + Overloaded { retry_after: Option }, + Misrouted, + Unavailable, + Refused, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Anchor { + pub height: u64, + pub hash: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Script { + pub script: String, + pub required_from: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scope { + pub source_id: String, + pub generation: u64, + pub scope_id: String, + pub anchor: Anchor, + pub scripts: Vec