diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e3b11f..05c468e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,28 @@ not always match the Git tags; section versions follow the tags. ## [Unreleased] +### Changed + +- `integrity-blob`: the `blob-azure` backend moves from the legacy `azure_storage` / + `azure_storage_blobs` 0.21.0 crates to `object_store` 0.14.2 (Apache Arrow). + `AzureBlob::new` and the FFI's `ig_blob_store_azure_blob_new` keep the same + arguments and still authenticate with the storage account key. A missing blob + still reads as `None`, and puts still store a block blob with content type + `text/plain`. Requests are now retried on server errors and timeouts, connecting + times out after 5 seconds, and a key that isn't valid base64 fails at `init` + rather than on the first request. + ([#55](https://github.com/eqtylab/integrity/pull/55)) + +### Security + +- `blob-azure` no longer depends on `azure_core` 0.21, which logs each request's + `authorization` header (the Shared Key signature) at debug and trace level + (RUSTSEC-2026-0275). The legacy Azure SDK is unsupported and won't be fixed + upstream. If you ran the Azure backend with debug or trace logging enabled for the + Azure crates, treat those logs as holding credentials and rotate the storage + account key. `signer-akv` still depends on `azure_core` 0.21. + ([#55](https://github.com/eqtylab/integrity/pull/55)) + ## [0.0.19] - 2026-10-16 ### Added diff --git a/Cargo.lock b/Cargo.lock index 7fb84b2..890295c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "RustyXML" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" - [[package]] name = "abnf" version = "0.13.0" @@ -879,18 +873,15 @@ dependencies = [ "dyn-clone", "futures", "getrandom 0.2.17", - "hmac 0.12.1", "http-types", "once_cell", "paste", "pin-project", - "quick-xml", "rand 0.8.5", "reqwest 0.12.28", "rustc_version", "serde", "serde_json", - "sha2 0.10.9", "time", "tracing", "url", @@ -932,62 +923,6 @@ dependencies = [ "time", ] -[[package]] -name = "azure_storage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" -dependencies = [ - "RustyXML", - "async-lock", - "async-trait", - "azure_core", - "bytes", - "serde", - "serde_derive", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_storage_blobs" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" -dependencies = [ - "RustyXML", - "azure_core", - "azure_storage", - "azure_svc_blobstorage", - "bytes", - "futures", - "serde", - "serde_derive", - "serde_json", - "time", - "tracing", - "url", - "uuid", -] - -[[package]] -name = "azure_svc_blobstorage" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" -dependencies = [ - "azure_core", - "bytes", - "futures", - "log", - "once_cell", - "serde", - "serde_json", - "time", -] - [[package]] name = "backtrace" version = "0.3.76" @@ -1055,6 +990,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1169,7 +1110,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.4.2", - "cpufeatures", + "cpufeatures 0.2.17", "memmap2", "rayon-core", ] @@ -1377,6 +1318,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.43" @@ -1644,6 +1596,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -1831,7 +1792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", @@ -2014,7 +1975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.114", ] [[package]] @@ -2923,11 +2884,23 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + [[package]] name = "gimli" version = "0.32.3" @@ -3871,8 +3844,6 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-s3", - "azure_storage", - "azure_storage_blobs", "blake3", "bytes", "cid 0.11.3", @@ -3881,6 +3852,7 @@ dependencies = [ "google-cloud-storage", "iroh-blake3", "log", + "object_store", ] [[package]] @@ -4190,6 +4162,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -4608,7 +4589,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -5322,6 +5303,44 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.15.0", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.3", + "reqwest 0.13.1", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.17", + "tokio", + "tracing", + "url", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -5835,7 +5854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.114", @@ -5852,9 +5871,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5931,6 +5950,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.6.2" @@ -5978,6 +6003,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -6036,6 +6072,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_hc" version = "0.2.0" @@ -6235,6 +6277,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -6853,7 +6896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -6865,7 +6908,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.9.0", "opaque-debug", ] @@ -6877,7 +6920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] diff --git a/integrity-blob/Cargo.toml b/integrity-blob/Cargo.toml index 3f6bf09..31f8733 100644 --- a/integrity-blob/Cargo.toml +++ b/integrity-blob/Cargo.toml @@ -9,7 +9,7 @@ blob-local = [] blob-memory = [] blob-s3 = ["dep:aws-config", "dep:aws-sdk-s3"] blob-gcs = ["dep:bytes", "dep:google-cloud-storage"] -blob-azure = ["dep:azure_storage", "dep:azure_storage_blobs"] +blob-azure = ["dep:object_store"] blob-all = ["blob-local", "blob-memory", "blob-s3", "blob-gcs", "blob-azure"] [dependencies] @@ -39,14 +39,6 @@ default-features = false features = ["behavior-version-latest", "default-https-client", "rt-tokio", "sigv4a"] optional = true -[target.'cfg(not(target_arch = "wasm32"))'.dependencies.azure_storage] -version = "0.21" -optional = true - -[target.'cfg(not(target_arch = "wasm32"))'.dependencies.azure_storage_blobs] -version = "0.21" -optional = true - [target.'cfg(not(target_arch = "wasm32"))'.dependencies.bytes] version = "1.5" optional = true @@ -55,5 +47,11 @@ optional = true version = "1.7" optional = true +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.object_store] +version = "0.14" +default-features = false +features = ["azure"] +optional = true + [dev-dependencies] futures-executor = "0.3" diff --git a/integrity-blob/src/blob_store/azure_blob.rs b/integrity-blob/src/blob_store/azure_blob.rs index 9b56e83..d638ae1 100644 --- a/integrity-blob/src/blob_store/azure_blob.rs +++ b/integrity-blob/src/blob_store/azure_blob.rs @@ -1,8 +1,11 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; -use azure_storage::{prelude::*, ErrorKind}; -use azure_storage_blobs::prelude::*; use log::{debug, trace, warn}; +use object_store::{ + azure::{MicrosoftAzure, MicrosoftAzureBuilder}, + path::Path, + Attribute, Attributes, ClientOptions, Error, ObjectStore, ObjectStoreExt, +}; use crate::blob_store::{calc_and_validate_cid, BlobStore}; @@ -14,7 +17,7 @@ pub struct AzureBlob { key: String, container: String, - client: Option, + client: Option, } impl AzureBlob { @@ -37,64 +40,52 @@ impl AzureBlob { #[async_trait] impl BlobStore for AzureBlob { async fn init(&mut self) -> Result<()> { - let storage_credentials = StorageCredentials::access_key(&self.account, self.key.clone()); - - let blob_client = ClientBuilder::new(&self.account, storage_credentials); - - self.client = blob_client.into(); + // object_store limits a whole request to 30s by default, which a large blob + // can exceed. Keep its connect timeout and retries, but not that limit. + let client_options = ClientOptions::new().with_timeout_disabled(); + + let client = MicrosoftAzureBuilder::new() + .with_account(&self.account) + .with_access_key(&self.key) + .with_container_name(&self.container) + .with_client_options(client_options) + .build()?; + + self.client = Some(client); Ok(()) } /// Check if a CID exists in the store async fn exists(&self, cid: &str) -> Result { - let client = self.client.clone().ok_or(anyhow!("client not init"))?; - - let exists = client - .blob_client(self.container.as_str(), cid) - .exists() - .await?; + let client = self.client.as_ref().ok_or(anyhow!("client not init"))?; - Ok(exists) + match client.head(&Path::from(cid)).await { + Ok(_) => Ok(true), + Err(Error::NotFound { .. }) => Ok(false), + Err(e) => Err(e.into()), + } } /// Get a blob from the store async fn get(&self, cid: &str) -> Result>> { let client = self .client - .clone() + .as_ref() .ok_or(anyhow!("client not initialized"))?; - match client - .blob_client(self.container.as_str(), cid) - .get_content() - .await - { - Ok(content) => Ok(Some(content)), - Err(e) => match e.kind() { - ErrorKind::HttpResponse { status, error_code } if u16::from(*status) == 404 => { - debug!( - "Blob '{cid}' not found in Azure Blob Storage. status=404 error_code={}", - error_code.as_deref().unwrap_or("unknown") - ); - Ok(None) - } - ErrorKind::HttpResponse { status, error_code } => { - let status_code = u16::from(*status); - warn!( - "Azure Blob Storage returned an HTTP error downloading blob '{cid}'. status={} error_code={} error={e}", - status_code, - error_code.as_deref().unwrap_or("unknown") - ); - Err(e.into()) - } - kind => { - warn!( - "Azure Blob Storage returned a non-HTTP error downloading blob '{cid}'. kind={} error={e}", - kind - ); - Err(e.into()) - } - }, + let location = Path::from(cid); + let content = async { client.get(&location).await?.bytes().await }; + + match content.await { + Ok(content) => Ok(Some(content.to_vec())), + Err(Error::NotFound { .. }) => { + debug!("Blob '{cid}' not found in Azure Blob Storage. status=404"); + Ok(None) + } + Err(e) => { + warn!("Azure Blob Storage returned an error downloading blob '{cid}'. error={e}"); + Err(e.into()) + } } } @@ -103,12 +94,13 @@ impl BlobStore for AzureBlob { let cid = calc_and_validate_cid(&blob, multicodec_code, cid)?; trace!("put {cid}. blob size: {}", blob.len()); - let client = self.client.clone().ok_or(anyhow!("client not init"))?; + let client = self.client.as_ref().ok_or(anyhow!("client not init"))?; + + let mut attributes = Attributes::new(); + attributes.insert(Attribute::ContentType, "text/plain".into()); client - .blob_client(self.container.as_str(), &cid) - .put_block_blob(blob) - .content_type("text/plain") + .put_opts(&Path::from(cid.as_str()), blob.into(), attributes.into()) .await?; Ok(cid)