From 441ec4217268bed05b5b33e628527180de6abea6 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 09:15:04 -0400 Subject: [PATCH 1/3] Update to arrow-rs main (arrow 60 pre-release) Pin arrow/parquet crates to arrow-rs rev 901e69f913bb8a1738009218898959e8f4cd7ec8 via [patch.crates-io] until arrow 60.0.0 is released. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 225 ++++++++++++++++++++++++++++++++++++++++------------- Cargo.toml | 21 ++++- 2 files changed, 189 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5365be12a1061..1f1e5271d82c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,8 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-arith", "arrow-array", @@ -182,14 +181,13 @@ dependencies = [ "arrow-select", "arrow-string", "half", - "rand 0.9.4", + "rand 0.10.1", ] [[package]] name = "arrow-arith" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -202,8 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-buffer", @@ -222,8 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -234,7 +230,7 @@ dependencies = [ "flate2", "indexmap 2.14.0", "liblzma", - "rand 0.9.4", + "rand 0.10.1", "serde", "serde_json", "snap", @@ -246,8 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "bytes", "half", @@ -258,8 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -277,11 +271,20 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cmp" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-csv" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-cast", @@ -295,8 +298,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-buffer", "arrow-schema", @@ -308,8 +310,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-arith", "arrow-array", @@ -335,8 +336,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,8 +351,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,11 +375,11 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -389,8 +388,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,8 +400,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "bitflags", "serde", @@ -414,12 +411,12 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -428,8 +425,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -522,9 +518,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" dependencies = [ "num-traits", ] @@ -1381,6 +1377,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1515,6 +1521,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -3310,8 +3326,6 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.4", - "rand_distr", "zerocopy", ] @@ -3856,6 +3870,55 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4382,14 +4445,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -4398,14 +4463,14 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5 0.11.0", + "nix", "parking_lot", "percent-encoding", "quick-xml", "rand 0.10.1", "reqwest", - "ring", "rustls-pki-types", "serde", "serde_json", @@ -4417,6 +4482,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -4497,8 +4563,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-array", @@ -4932,9 +4997,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -4966,6 +5031,7 @@ version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -5254,9 +5320,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -5275,11 +5341,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", + "rustls-platform-verifier", "sync_wrapper", "tokio", "tokio-rustls", @@ -5414,6 +5477,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -5784,6 +5874,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -5848,6 +5948,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqllogictest" version = "0.29.1" @@ -6988,9 +7094,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -7031,6 +7137,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "2.1.1" diff --git a/Cargo.toml b/Cargo.toml index 65670d895e1a8..1bbf056f9b8f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ arrow-avro = { version = "59.2.0", default-features = false, features = [ arrow-buffer = { version = "59.2.0", default-features = false } arrow-data = { version = "59.2.0", default-features = false } arrow-flight = { version = "59.2.0", features = [ - "flight-sql-experimental", + "flight-sql", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. @@ -176,7 +176,7 @@ liblzma = { version = "0.4.6", features = ["static"] } log = "^0.4" memchr = "2.8.1" num-traits = { version = "0.2" } -object_store = { version = "0.13.2", default-features = false } +object_store = { version = "0.14.1", default-features = false } parking_lot = "0.12" parquet = { version = "59.2.0", default-features = false, features = [ "arrow", @@ -359,3 +359,20 @@ debug = false debug-assertions = false strip = "debuginfo" incremental = false + +## Temporary arrow-rs patch until 60.0.0 is released + +[patch.crates-io] +arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-avro = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-data = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-select = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-string = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-ord = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } From d58f5798d30bdee041e5f8b7f36dae331e81fff6 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 10:19:41 -0400 Subject: [PATCH 2/3] Fix compilation for arrow-rs main (Metadata type, rand 0.10, object_store 0.14) - Adapt to the new arrow_schema::Metadata type (was HashMap): DFSchema::metadata()/ExprSchema::metadata() now return &Metadata, DFSchema constructors take impl Into, SchemaFieldMetadata is now an alias for Metadata, and check_metadata_with_storage_equal/format_type_and_metadata accept Option<&Metadata> - Add FieldMetadata <-> Metadata conversions and a DFHeapSize impl for Metadata - Replace deprecated fb_to_schema with try_fb_to_schema - Rework two benches that mixed DataFusion's rand 0.9 with arrow's rand 0.10 - Add new object_store 0.14 GetResult/ListResult extensions field - Update map field name expectations for new arrow spec defaults (key/value) Co-Authored-By: Claude Fable 5 --- datafusion/catalog-listing/src/helpers.rs | 4 +- datafusion/catalog-listing/src/table.rs | 4 +- datafusion/common/src/dfschema.rs | 17 ++--- datafusion/common/src/heap_size.rs | 15 ++++- datafusion/common/src/metadata.rs | 39 ++++++++--- datafusion/common/src/nested_struct.rs | 12 ++-- datafusion/common/src/param_value.rs | 2 +- .../core/src/datasource/file_format/csv.rs | 1 + datafusion/core/tests/sql/path_partition.rs | 2 + .../user_defined/user_defined_aggregates.rs | 16 ++--- .../user_defined_scalar_functions.rs | 9 +-- .../user_defined_window_functions.rs | 9 +-- .../datasource-arrow/src/file_format.rs | 6 +- datafusion/datasource-arrow/src/source.rs | 7 +- .../datasource-parquet/src/file_format.rs | 4 +- datafusion/execution/src/cache/mod.rs | 6 +- datafusion/expr/src/expr.rs | 66 ++++++++++--------- datafusion/expr/src/logical_plan/builder.rs | 8 +-- datafusion/expr/src/logical_plan/plan.rs | 7 +- datafusion/ffi/src/record_batch_stream.rs | 4 +- datafusion/ffi/src/udaf/mod.rs | 8 +-- .../functions-aggregate/benches/array_agg.rs | 4 +- .../functions/src/core/with_metadata.rs | 11 +--- datafusion/optimizer/src/optimizer.rs | 5 +- .../optimizer/src/propagate_empty_relation.rs | 4 +- datafusion/optimizer/src/push_down_filter.rs | 4 +- .../src/simplify_expressions/udf_preimage.rs | 6 +- datafusion/physical-expr/benches/case_when.rs | 9 ++- datafusion/physical-expr/src/planner.rs | 9 ++- datafusion/physical-plan/src/joins/utils.rs | 8 +-- datafusion/physical-plan/src/union.rs | 4 +- datafusion/proto-common/src/from_proto/mod.rs | 8 ++- datafusion/proto-common/src/to_proto/mod.rs | 8 +-- datafusion/proto/src/logical_plan/to_proto.rs | 6 +- .../tests/cases/roundtrip_logical_plan.rs | 8 +-- datafusion/proto/tests/cases/serialize.rs | 6 +- datafusion/sql/src/statement.rs | 4 +- datafusion/sql/tests/cases/params.rs | 12 ++-- datafusion/sql/tests/common/mod.rs | 6 +- datafusion/sqllogictest/src/test_context.rs | 5 +- 40 files changed, 192 insertions(+), 181 deletions(-) diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index 098f3d51ef911..b7cf6af3b7faa 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -30,7 +30,7 @@ use datafusion_expr::{BinaryExpr, Operator, lit, utils}; use arrow::{ array::AsArray, - datatypes::{DataType, Field}, + datatypes::{DataType, Field, Metadata}, record_batch::RecordBatch, }; use datafusion_expr::execution_props::ExecutionProps; @@ -423,7 +423,7 @@ pub async fn pruned_partition_list<'a>( .iter() .map(|(n, d)| Field::new(n, d.clone(), true)) .collect(), - Default::default(), + Metadata::new(), )?; Ok(objects diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index 46fab1f9cbff4..f2867912240c8 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -20,7 +20,7 @@ use crate::helpers::{ expr_applicable_for_cols, filter_partitioned_file, pruned_partition_list, }; use crate::{ListingOptions, ListingTableConfig}; -use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef}; +use arrow::datatypes::{Field, Metadata, Schema, SchemaBuilder, SchemaRef}; use async_trait::async_trait; use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::stats::Precision; @@ -1035,7 +1035,7 @@ impl ListingTable { .iter() .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) .collect(), - Default::default(), + Metadata::new(), )?; file_groups diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 262f1dcf619d9..a49879e466b26 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -18,7 +18,7 @@ //! DFSchema is an extended schema struct that DataFusion uses to provide support for //! fields with optional relation names. -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashSet}; use std::fmt::{Display, Formatter}; use std::hash::Hash; use std::sync::{Arc, LazyLock}; @@ -31,7 +31,7 @@ use crate::{ use arrow::compute::can_cast_types; use arrow::datatypes::{ - DataType, Field, FieldRef, Fields, Schema, SchemaBuilder, SchemaRef, + DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaBuilder, SchemaRef, }; /// A reference-counted reference to a [DFSchema]. @@ -153,7 +153,7 @@ impl DFSchema { /// Create a `DFSchema` from an Arrow schema where all the fields have a given qualifier pub fn new_with_metadata( qualified_fields: Vec<(Option, Arc)>, - metadata: HashMap, + metadata: impl Into, ) -> Result { let (qualifiers, fields): (Vec>, Vec>) = qualified_fields.into_iter().unzip(); @@ -172,7 +172,7 @@ impl DFSchema { /// Create a new `DFSchema` from a list of Arrow [Field]s pub fn from_unqualified_fields( fields: Fields, - metadata: HashMap, + metadata: impl Into, ) -> Result { let field_count = fields.len(); let schema = Arc::new(Schema::new_with_metadata(fields, metadata)); @@ -864,7 +864,7 @@ impl DFSchema { } /// Get metadata of this schema - pub fn metadata(&self) -> &HashMap { + pub fn metadata(&self) -> &Metadata { &self.inner.metadata } @@ -1179,7 +1179,7 @@ impl ToDFSchema for Vec { let field_count = self.len(); let schema = Schema { fields: self.into(), - metadata: HashMap::new(), + metadata: Metadata::new(), }; let dfschema = DFSchema { inner: schema.into(), @@ -1221,7 +1221,7 @@ pub trait ExprSchema: std::fmt::Debug { } /// Returns the column's optional metadata. - fn metadata(&self, col: &Column) -> Result<&HashMap> { + fn metadata(&self, col: &Column) -> Result<&Metadata> { Ok(self.field_from_column(col)?.metadata()) } @@ -1245,7 +1245,7 @@ impl + std::fmt::Debug> ExprSchema for P { self.as_ref().data_type(col) } - fn metadata(&self, col: &Column) -> Result<&HashMap> { + fn metadata(&self, col: &Column) -> Result<&Metadata> { ExprSchema::metadata(self.as_ref(), col) } @@ -1379,6 +1379,7 @@ pub fn qualified_name(qualifier: Option<&TableReference>, name: &str) -> String #[cfg(test)] mod tests { use crate::assert_contains; + use std::collections::HashMap; use super::*; diff --git a/datafusion/common/src/heap_size.rs b/datafusion/common/src/heap_size.rs index 037f807fce9d8..7f774fd5a6826 100644 --- a/datafusion/common/src/heap_size.rs +++ b/datafusion/common/src/heap_size.rs @@ -47,7 +47,7 @@ use arrow::array::{ }; use arrow::datatypes::{ DataType, Field, Fields, IntervalDayTime, IntervalMonthDayNano, IntervalUnit, - TimeUnit, UnionFields, UnionMode, i256, + Metadata, TimeUnit, UnionFields, UnionMode, i256, }; use chrono::{DateTime, Utc}; use half::f16; @@ -396,6 +396,19 @@ impl DFHeapSize for UnionFields { } } +impl DFHeapSize for Metadata { + fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize { + // `Metadata` does not expose its underlying reference-counted map, so + // this approximates the `BTreeMap` entries' sizes and cannot dedupe + // instances that share the same allocation. + self.iter() + .map(|(k, v)| { + size_of::<(String, String)>() + k.heap_size(ctx) + v.heap_size(ctx) + }) + .sum() + } +} + impl DFHeapSize for Field { fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize { self.name().heap_size(ctx) diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rs index d6d8fb7b0ed0c..60c072a2111ec 100644 --- a/datafusion/common/src/metadata.rs +++ b/datafusion/common/src/metadata.rs @@ -17,7 +17,7 @@ use std::{collections::BTreeMap, sync::Arc}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use hashbrown::HashMap; use crate::{DataFusionError, ScalarValue, error::_plan_err}; @@ -84,14 +84,8 @@ impl From for ScalarAndMetadata { /// Returns a planning error with suitably formatted type representations if /// actual and expected do not compare to equal. pub fn check_metadata_with_storage_equal( - actual: ( - &DataType, - Option<&std::collections::HashMap>, - ), - expected: ( - &DataType, - Option<&std::collections::HashMap>, - ), + actual: (&DataType, Option<&Metadata>), + expected: (&DataType, Option<&Metadata>), what: &str, context: &str, ) -> Result<(), DataFusionError> { @@ -131,7 +125,7 @@ pub fn check_metadata_with_storage_equal( /// renderings. pub fn format_type_and_metadata( data_type: &DataType, - metadata: Option<&std::collections::HashMap>, + metadata: Option<&Metadata>, ) -> String { match metadata { Some(metadata) if !metadata.is_empty() => { @@ -316,6 +310,13 @@ impl FieldMetadata { .collect() } + /// Convert this `FieldMetadata` into an arrow [`Metadata`] + /// + /// This is cheap: both types share the same `Arc` representation. + pub fn to_metadata(&self) -> Metadata { + Metadata::from(Arc::clone(&self.inner)) + } + /// Updates the metadata on the Field with this metadata, if it is not empty. pub fn add_to_field(&self, field: Field) -> Field { if self.inner.is_empty() { @@ -336,6 +337,24 @@ impl FieldMetadata { } } +impl From<&FieldMetadata> for Metadata { + fn from(value: &FieldMetadata) -> Self { + value.to_metadata() + } +} + +impl From for FieldMetadata { + fn from(value: Metadata) -> Self { + Self::new(value.into()) + } +} + +impl From<&Metadata> for FieldMetadata { + fn from(value: &Metadata) -> Self { + Self::from(value.clone()) + } +} + impl From<&Field> for FieldMetadata { fn from(field: &Field) -> Self { Self::new_from_field(field) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index e915b91b911cc..da994f7a95196 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -1123,8 +1123,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1148,8 +1148,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1176,8 +1176,8 @@ mod tests { assert!(map.is_null(1)); let map0 = map.value(0); let entries = map0.as_any().downcast_ref::().unwrap(); - let keys = get_column_as!(entries, "keys", StringArray); - let vals = get_column_as!(entries, "values", Int32Array); + let keys = get_column_as!(entries, "key", StringArray); + let vals = get_column_as!(entries, "value", Int32Array); assert_eq!(keys.value(0), "a"); assert_eq!(vals.value(0), 1); } diff --git a/datafusion/common/src/param_value.rs b/datafusion/common/src/param_value.rs index 0fac6b529eb0f..1979daa7e3755 100644 --- a/datafusion/common/src/param_value.rs +++ b/datafusion/common/src/param_value.rs @@ -64,7 +64,7 @@ impl ParamValues { check_metadata_with_storage_equal( ( &lit.value.data_type(), - lit.metadata.as_ref().map(|m| m.to_hashmap()).as_ref(), + lit.metadata.as_ref().map(|m| m.to_metadata()).as_ref(), ), (param_type.data_type(), Some(param_type.metadata())), "parameter", diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 90d7eb3b41388..b9cc06385996b 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -136,6 +136,7 @@ mod tests { }, range: Default::default(), attributes: Attributes::default(), + extensions: Default::default(), }) } diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs index 82a15eb401fc4..4fce490d3fce7 100644 --- a/datafusion/core/tests/sql/path_partition.rs +++ b/datafusion/core/tests/sql/path_partition.rs @@ -718,6 +718,7 @@ impl ObjectStore for MirroringObjectStore { payload, meta, attributes: Attributes::default(), + extensions: Default::default(), }) } @@ -789,6 +790,7 @@ impl ObjectStore for MirroringObjectStore { Ok(ListResult { common_prefixes: common_prefixes.into_iter().collect(), objects, + extensions: Default::default(), }) } diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs index 323925bcfaf82..12e72f6091c8a 100644 --- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs +++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs @@ -30,7 +30,7 @@ use arrow::array::{ Array, AsArray, Int32Array, PrimitiveArray, StringArray, StructArray, UInt64Array, record_batch, types::UInt64Type, }; -use arrow::datatypes::{Fields, Schema}; +use arrow::datatypes::{Fields, Metadata, Schema}; use arrow_schema::FieldRef; use datafusion::common::test_util::batches_to_string; use datafusion::dataframe::DataFrame; @@ -1018,11 +1018,8 @@ async fn test_metadata_based_aggregate() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( @@ -1093,11 +1090,8 @@ async fn test_metadata_based_aggregate_as_window() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index b758aeb5209e8..474b86fb65aeb 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -25,7 +25,7 @@ use arrow::array::{ }; use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch}; use arrow::compute::kernels::numeric::add; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Metadata, Schema}; use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType}; use arrow_schema::{ArrowError, FieldRef, SchemaRef}; use datafusion::common::test_util::batches_to_string; @@ -1645,11 +1645,8 @@ async fn test_metadata_based_udf() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( schema, diff --git a/datafusion/core/tests/user_defined/user_defined_window_functions.rs b/datafusion/core/tests/user_defined/user_defined_window_functions.rs index afaf269ca1200..ea789bb9a4c39 100644 --- a/datafusion/core/tests/user_defined/user_defined_window_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_window_functions.rs @@ -22,7 +22,7 @@ use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, RecordBatch, StringArray, UInt64Array, record_batch, }; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Metadata, Schema}; use arrow_schema::FieldRef; use datafusion::common::test_util::batches_to_string; use datafusion::common::{Result, ScalarValue}; @@ -868,11 +868,8 @@ async fn test_metadata_based_window_fn() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( diff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rs index c50ad98dfca0b..c8ccc968482d1 100644 --- a/datafusion/datasource-arrow/src/file_format.rs +++ b/datafusion/datasource-arrow/src/file_format.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::ArrowError; -use arrow::ipc::convert::fb_to_schema; +use arrow::ipc::convert::try_fb_to_schema; use arrow::ipc::reader::{FileReader, StreamReader}; use arrow::ipc::writer::IpcWriteOptions; use arrow::ipc::{CompressionType, root_as_message}; @@ -481,7 +481,9 @@ async fn infer_stream_schema( let fb_schema = message.header_as_schema().ok_or_else(|| { ArrowError::IpcError("Unable to read IPC message schema".to_string()) })?; - let schema = fb_to_schema(fb_schema); + let schema = try_fb_to_schema(fb_schema).map_err(|err| { + ArrowError::IpcError(format!("Unable to convert IPC schema: {err:?}")) + })?; Ok(Arc::new(schema)) } diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs index dba99a9758886..bad2ba141a68a 100644 --- a/datafusion/datasource-arrow/src/source.rs +++ b/datafusion/datasource-arrow/src/source.rs @@ -179,7 +179,12 @@ impl FileOpener for ArrowFileOpener { })?; // build decoder according to footer & projection let schema = - arrow_ipc::convert::fb_to_schema(footer.schema().unwrap()); + arrow_ipc::convert::try_fb_to_schema(footer.schema().unwrap()) + .map_err(|err| { + exec_datafusion_err!( + "Unable to convert IPC schema: {err:?}" + ) + })?; let mut decoder = FileDecoder::new(schema.into(), footer.version()); if let Some(projection) = projection { decoder = decoder.with_projection(projection); diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs index 6358201c06fa5..bf35ad5a8c663 100644 --- a/datafusion/datasource-parquet/src/file_format.rs +++ b/datafusion/datasource-parquet/src/file_format.rs @@ -32,7 +32,7 @@ pub use crate::schema_coercion::{ pub use crate::sink::ParquetSink; -use arrow::datatypes::{Fields, Schema, SchemaRef}; +use arrow::datatypes::{Fields, Metadata, Schema, SchemaRef}; use datafusion_datasource::TableSchema; use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_datasource::file_sink_config::FileSinkConfig; @@ -266,7 +266,7 @@ fn clear_metadata( .fields() .iter() .map(|field| { - field.as_ref().clone().with_metadata(Default::default()) // clear meta + field.as_ref().clone().with_metadata(Metadata::new()) // clear meta }) .collect::(); Schema::new(fields) diff --git a/datafusion/execution/src/cache/mod.rs b/datafusion/execution/src/cache/mod.rs index f47a3f3ca49f3..2eba533e04b67 100644 --- a/datafusion/execution/src/cache/mod.rs +++ b/datafusion/execution/src/cache/mod.rs @@ -228,7 +228,7 @@ impl DFHeapSize for SchemaFingerprint { #[cfg(test)] mod schema_fingerprint_tests { use super::*; - use datafusion_common::arrow::datatypes::Field; + use datafusion_common::arrow::datatypes::{Field, Metadata}; fn fp(fields: Vec) -> SchemaFingerprint { SchemaFingerprint::from_schema(&Schema::new(fields)) @@ -263,13 +263,13 @@ mod schema_fingerprint_tests { let field_md = SchemaFingerprint::from_schema(&Schema::new(vec![ Field::new("id", DataType::Int64, false) - .with_metadata([("note".to_string(), "x".to_string())].into()), + .with_metadata(Metadata::new().with("note", "x")), ])); assert_eq!(plain, field_md, "field metadata must be ignored"); let schema_md = SchemaFingerprint::from_schema( &Schema::new(vec![Field::new("id", DataType::Int64, false)]) - .with_metadata([("k".to_string(), "v".to_string())].into()), + .with_metadata(Metadata::new().with("k", "v")), ); assert_eq!(plain, schema_md, "schema metadata must be ignored"); } diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index f9c0662e682e8..82186f569be85 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -32,7 +32,7 @@ use crate::type_coercion::functions::value_fields_with_higher_order_udf; use crate::{AggregateUDF, LambdaParametersProgress, ValueOrLambda, Volatility}; use crate::{ExprSchemable, Operator, Signature, WindowFrame, WindowUDF}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use datafusion_common::cse::{HashNode, NormalizeEq, Normalizeable}; use datafusion_common::datatype::DataTypeExt; use datafusion_common::metadata::format_type_and_metadata; @@ -622,7 +622,7 @@ impl<'a> TreeNodeContainer<'a, Self> for Expr { /// See the [default_column_values.rs] example implementation. /// /// [default_column_values.rs]: https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs -pub type SchemaFieldMetadata = std::collections::HashMap; +pub type SchemaFieldMetadata = Metadata; /// Intersects multiple metadata instances for UNION operations. /// @@ -648,7 +648,7 @@ pub type SchemaFieldMetadata = std::collections::HashMap; pub fn intersect_metadata_for_union<'a>( metadatas: impl IntoIterator, ) -> SchemaFieldMetadata { - let mut intersected: Option = None; + let mut intersected: Option> = None; for metadata in metadatas { // Skip empty metadata (e.g. from NULL literals or computed expressions) @@ -658,7 +658,12 @@ pub fn intersect_metadata_for_union<'a>( } match &mut intersected { None => { - intersected = Some(metadata.clone()); + intersected = Some( + metadata + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ); } Some(current) => { // Only keep keys that exist in both with the same value @@ -667,7 +672,7 @@ pub fn intersect_metadata_for_union<'a>( } } - intersected.unwrap_or_default() + intersected.map(Metadata::from).unwrap_or_default() } /// UNNEST expression. @@ -3901,7 +3906,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -3949,7 +3954,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4003,7 +4008,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4056,7 +4061,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4157,9 +4162,8 @@ mod test { fn infer_placeholder_with_metadata() { // name == $1, where name is a non-nullable string let schema = Arc::new(Schema::new(vec![ - Field::new("name", DataType::Utf8, false).with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ), + Field::new("name", DataType::Utf8, false) + .with_metadata(Metadata::new().with("some_key", "some_value")), ])); let df_schema = DFSchema::try_from(schema).unwrap(); @@ -4593,53 +4597,53 @@ mod test { mod intersect_metadata_tests { use super::super::intersect_metadata_for_union; - use std::collections::HashMap; + use arrow::datatypes::Metadata; #[test] fn all_branches_same_metadata() { - let m1 = HashMap::from([("key".into(), "val".into())]); - let m2 = HashMap::from([("key".into(), "val".into())]); + let m1 = Metadata::new().with("key", "val"); + let m2 = Metadata::new().with("key", "val"); let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn conflicting_metadata_dropped() { - let m1 = HashMap::from([("key".into(), "a".into())]); - let m2 = HashMap::from([("key".into(), "b".into())]); + let m1 = Metadata::new().with("key", "a"); + let m2 = Metadata::new().with("key", "b"); let result = intersect_metadata_for_union([&m1, &m2]); assert!(result.is_empty()); } #[test] fn empty_metadata_branch_skipped() { - let m1 = HashMap::from([("key".into(), "val".into())]); - let m2 = HashMap::new(); // e.g. NULL literal + let m1 = Metadata::new().with("key", "val"); + let m2 = Metadata::new(); // e.g. NULL literal let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn empty_metadata_first_branch_skipped() { - let m1 = HashMap::new(); - let m2 = HashMap::from([("key".into(), "val".into())]); + let m1 = Metadata::new(); + let m2 = Metadata::new().with("key", "val"); let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn all_branches_empty_metadata() { - let m1: HashMap = HashMap::new(); - let m2: HashMap = HashMap::new(); + let m1 = Metadata::new(); + let m2 = Metadata::new(); let result = intersect_metadata_for_union([&m1, &m2]); assert!(result.is_empty()); } #[test] fn mixed_empty_and_conflicting() { - let m1 = HashMap::from([("key".into(), "a".into())]); - let m2 = HashMap::new(); - let m3 = HashMap::from([("key".into(), "b".into())]); + let m1 = Metadata::new().with("key", "a"); + let m2 = Metadata::new(); + let m3 = Metadata::new().with("key", "b"); let result = intersect_metadata_for_union([&m1, &m2, &m3]); // m2 is skipped; m1 and m3 conflict → dropped assert!(result.is_empty()); @@ -4647,9 +4651,7 @@ mod test { #[test] fn no_inputs() { - let result = intersect_metadata_for_union(std::iter::empty::< - &HashMap, - >()); + let result = intersect_metadata_for_union(std::iter::empty::<&Metadata>()); assert!(result.is_empty()); } } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 4b728fe4474bc..48fea79493bf6 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -1765,12 +1765,8 @@ pub fn build_join_schema( _ => (right, left), }; - let metadata = schema1 - .metadata() - .clone() - .into_iter() - .chain(schema2.metadata().clone()) - .collect(); + let mut metadata = schema1.metadata().clone(); + metadata.extend(schema2.metadata().clone()); let dfschema = DFSchema::new_with_metadata(qualified_fields, metadata)?; dfschema.with_functional_dependencies(func_dependencies) diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 1a141ea52a13a..129be81faa533 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -53,7 +53,7 @@ use crate::{ use crate::statistics::StatisticsRequest; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef}; use datafusion_common::cse::{NormalizeEq, Normalizeable}; use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType}; use datafusion_common::metadata::check_metadata_with_storage_equal; @@ -3269,8 +3269,7 @@ impl Union { inputs: &[Arc], loose_types: bool, ) -> Result { - type FieldData<'a> = - (&'a DataType, bool, Vec<&'a HashMap>, usize); + type FieldData<'a> = (&'a DataType, bool, Vec<&'a Metadata>, usize); let mut cols: Vec<(&str, FieldData)> = Vec::new(); for input in inputs.iter() { for field in input.schema().fields() { @@ -5658,7 +5657,7 @@ mod tests { let schema_with_metadata = || { DFSchema::from_unqualified_fields( vec![Field::new("count", DataType::Int64, false)].into(), - [("key".to_string(), "value".to_string())].into(), + Metadata::new().with("key", "value"), ) .unwrap() }; diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs index 5a92cbfe5fe78..2866460c1825c 100644 --- a/datafusion/ffi/src/record_batch_stream.rs +++ b/datafusion/ffi/src/record_batch_stream.rs @@ -219,7 +219,7 @@ mod tests { use std::sync::Arc; use arrow::array::record_batch; - use arrow::datatypes::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Metadata, Schema}; use datafusion::error::Result; use datafusion::execution::SendableRecordBatchStream; use datafusion::test_util::bounded_stream; @@ -274,7 +274,7 @@ mod tests { .schema() .as_ref() .clone() - .with_metadata([("some_key".to_owned(), "some_value".to_owned())].into()) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(); let rb = rb.with_schema(schema)?; diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs index b3a087e5d0022..0fa15d5aacf39 100644 --- a/datafusion/ffi/src/udaf/mod.rs +++ b/datafusion/ffi/src/udaf/mod.rs @@ -657,9 +657,8 @@ impl From for FFI_AggregateOrderSensitivity { #[cfg(test)] mod tests { - use std::collections::HashMap; - use arrow::datatypes::Schema; + use arrow::datatypes::{Metadata, Schema}; use datafusion::common::create_array; use datafusion::functions_aggregate::sum::Sum; use datafusion::physical_expr::PhysicalSortExpr; @@ -776,10 +775,7 @@ mod tests { let foreign_udaf: Arc = (&local_udaf).into(); let foreign_udaf = AggregateUDF::new_from_shared_impl(foreign_udaf); - let metadata: HashMap = - [("a_key".to_string(), "a_value".to_string())] - .into_iter() - .collect(); + let metadata = Metadata::new().with("a_key", "a_value"); let input_field = Arc::new( Field::new("a", DataType::Float64, false).with_metadata(metadata.clone()), ); diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index d7e5a511078a5..4d88df4200d7c 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -33,7 +33,6 @@ use arrow::buffer::OffsetBuffer; use arrow::util::bench_util::create_primitive_array; use rand::Rng; use rand::SeedableRng; -use rand::distr::{Distribution, StandardUniform}; use rand::prelude::StdRng; /// Returns fixed seedable RNG @@ -65,8 +64,7 @@ pub fn create_list_array( zero_length_lists_probability: f32, ) -> ListArray where - T: ArrowPrimitiveType, - StandardUniform: Distribution, + T: ArrowPrimitiveType, { let mut nulls_builder = NullBufferBuilder::new(size); let mut rng = StdRng::seed_from_u64(42); diff --git a/datafusion/functions/src/core/with_metadata.rs b/datafusion/functions/src/core/with_metadata.rs index 481ed713ed7ad..189f6360576e2 100644 --- a/datafusion/functions/src/core/with_metadata.rs +++ b/datafusion/functions/src/core/with_metadata.rs @@ -157,7 +157,7 @@ impl ScalarUDFImpl for WithMetadataFunc { #[cfg(test)] mod tests { use super::*; - use arrow::datatypes::Field; + use arrow::datatypes::{Field, Metadata}; use datafusion_common::ScalarValue; use std::sync::Arc; @@ -197,14 +197,7 @@ mod tests { fn merges_existing_metadata_and_overwrites_on_collision() { let udf = WithMetadataFunc::new(); let mut existing = Field::new("x", DataType::Float64, false); - existing.set_metadata( - [ - ("keep".to_string(), "yes".to_string()), - ("unit".to_string(), "old".to_string()), - ] - .into_iter() - .collect(), - ); + existing.set_metadata(Metadata::new().with("keep", "yes").with("unit", "old")); let input: FieldRef = Arc::new(existing); let k = str_lit("unit"); let v = str_lit("new"); diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index db7ad8475273a..7a459cd7db4e0 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -783,6 +783,8 @@ fn assert_valid_optimization( mod tests { use std::sync::{Arc, Mutex}; + use arrow::datatypes::Metadata; + use datafusion_common::tree_node::Transformed; use datafusion_common::{ Column, DFSchema, DFSchemaRef, DataFusionError, Result, assert_contains, plan_err, @@ -962,8 +964,7 @@ mod tests { .iter() .enumerate() .map(|(i, (qualifier, field))| { - let metadata = - [("key".into(), format!("value {i}"))].into_iter().collect(); + let metadata = Metadata::new().with("key", format!("value {i}")); let new_arrow_field = field.as_ref().clone().with_metadata(metadata); (qualifier.cloned(), Arc::new(new_arrow_field)) diff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rs index 18ddc361a0692..49e358a854a2f 100644 --- a/datafusion/optimizer/src/propagate_empty_relation.rs +++ b/datafusion/optimizer/src/propagate_empty_relation.rs @@ -348,7 +348,7 @@ fn has_empty_grouping_set(group_expr: &[Expr]) -> bool { #[cfg(test)] mod tests { - use arrow::datatypes::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Metadata, Schema}; use datafusion_common::{Column, DFSchema}; use datafusion_expr::logical_plan::table_scan; @@ -800,7 +800,7 @@ mod tests { produce_one_row: false, schema: Arc::new(DFSchema::from_unqualified_fields( fields.into(), - Default::default(), + Metadata::new(), )?), }); diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index cf54ae254746d..628a8ead99e57 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1437,7 +1437,7 @@ mod tests { use std::cmp::Ordering; use std::fmt::{Debug, Formatter}; - use arrow::datatypes::{Field, Schema, SchemaRef}; + use arrow::datatypes::{Field, Metadata, Schema, SchemaRef}; use async_trait::async_trait; use datafusion_common::{DFSchemaRef, DataFusionError, ScalarValue}; @@ -4263,7 +4263,7 @@ mod tests { let schema = Arc::new( DFSchema::new_with_metadata( vec![(None, Field::new("a", DataType::Int64, false).into())], - Default::default(), + Metadata::new(), ) .unwrap(), ); diff --git a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs index d888a54d56574..c71d17d3034ab 100644 --- a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs +++ b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs @@ -71,7 +71,7 @@ mod test { use std::sync::Arc; - use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::{DataType, Field, Metadata}; use datafusion_common::{DFSchema, DFSchemaRef, Result, ScalarValue}; use datafusion_expr::{ ColumnarValue, Expr, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, @@ -201,7 +201,7 @@ mod test { Arc::new( DFSchema::from_unqualified_fields( vec![Field::new("x", DataType::Int32, true)].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ) @@ -215,7 +215,7 @@ mod test { Field::new("y", DataType::Int32, false), ] .into(), - Default::default(), + Metadata::new(), ) .unwrap(), ) diff --git a/datafusion/physical-expr/benches/case_when.rs b/datafusion/physical-expr/benches/case_when.rs index 33931a2ba98e4..88239149e042c 100644 --- a/datafusion/physical-expr/benches/case_when.rs +++ b/datafusion/physical-expr/benches/case_when.rs @@ -18,7 +18,6 @@ use arrow::array::{Array, ArrayRef, Int32Array, Int32Builder, StringArray}; use arrow::datatypes::{ArrowNativeTypeOp, Field, Schema}; use arrow::record_batch::RecordBatch; -use arrow::util::test_util::seedable_rng; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use datafusion_common::ScalarValue; use datafusion_expr::Operator; @@ -28,7 +27,13 @@ use itertools::Itertools; use rand::distr::Alphanumeric; use rand::distr::uniform::SampleUniform; use rand::rngs::StdRng; -use rand::{Rng, RngCore}; +use rand::{Rng, RngCore, SeedableRng}; + +/// Returns a fixed-seed RNG using this crate's `rand` version (arrow's +/// `test_util::seedable_rng` returns its own `rand` version's `StdRng`) +fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} use std::fmt::{Display, Formatter}; use std::hint::black_box; use std::ops::Range; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index f80d1b15bdc59..e784805e2f55e 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -743,7 +743,7 @@ pub fn logical2physical(expr: &Expr, schema: &Schema) -> Arc { #[cfg(test)] mod tests { use arrow::array::{ArrayRef, BooleanArray, RecordBatch, StringArray}; - use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::{DataType, Field, Metadata}; use datafusion_expr::col; use super::*; @@ -803,7 +803,7 @@ mod tests { let schema = test_cast_schema(); let target_field = Arc::new( Field::new("cast_target", DataType::Int64, true) - .with_metadata([("target_meta".to_string(), "1".to_string())].into()), + .with_metadata(Metadata::new().with("target_meta", "1")), ); let cast_expr = Expr::Cast(Cast::new_from_field( Box::new(col("a")), @@ -841,9 +841,8 @@ mod tests { fn test_cast_lowering_preserves_same_type_field_semantics() -> Result<()> { let schema = test_cast_schema(); let target_field = Arc::new( - Field::new("same_type_cast", DataType::Int32, true).with_metadata( - [("target_meta".to_string(), "same-type".to_string())].into(), - ), + Field::new("same_type_cast", DataType::Int32, true) + .with_metadata(Metadata::new().with("target_meta", "same-type")), ); let cast_expr = Expr::Cast(Cast::new_from_field( Box::new(col("a")), diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs index 20467a7ec5e33..3bbd3da322304 100644 --- a/datafusion/physical-plan/src/joins/utils.rs +++ b/datafusion/physical-plan/src/joins/utils.rs @@ -338,12 +338,8 @@ pub fn build_join_schema( _ => (right, left), }; - let metadata = schema1 - .metadata() - .clone() - .into_iter() - .chain(schema2.metadata().clone()) - .collect(); + let mut metadata = schema1.metadata().clone(); + metadata.extend(schema2.metadata().clone()); (fields.finish().with_metadata(metadata), column_indices) } diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index c1cc5da31abaf..bf70af1ae89fb 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -46,7 +46,7 @@ use crate::statistics::{ChildStats, StatisticsArgs}; use crate::stream::ObservedStream; use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count}; -use arrow::datatypes::{Field, Schema, SchemaRef}; +use arrow::datatypes::{Field, Metadata, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::config::ConfigOptions; use datafusion_common::stats::NdvFallback; @@ -963,7 +963,7 @@ fn union_schema(inputs: &[Arc]) -> Result { }) .collect::>(); - let all_metadata_merged = inputs + let all_metadata_merged: Metadata = inputs .iter() .flat_map(|i| i.schema().metadata().clone().into_iter()) .collect(); diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index 169ff7f3d9ff2..df583112672aa 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -28,7 +28,7 @@ use arrow::datatypes::{ TimeUnit, UnionFields, UnionMode, i256, }; use arrow::ipc::{ - convert::fb_to_schema, + convert::try_fb_to_schema, reader::{read_dictionary, read_record_batch}, root_as_message, writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions}, @@ -461,7 +461,11 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue { .to_string(), ) })?; - fb_to_schema(ipc_schema) + try_fb_to_schema(ipc_schema).map_err(|e| { + Error::General(format!( + "Error converting IPC schema while deserializing nested ScalarValue: {e}" + )) + })? }; let message = root_as_message(ipc_message.as_slice()).map_err(|e| { diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs index 360981746585b..601efbc3d03a5 100644 --- a/datafusion/proto-common/src/to_proto/mod.rs +++ b/datafusion/proto-common/src/to_proto/mod.rs @@ -99,7 +99,7 @@ impl TryFrom<&Field> for protobuf::Field { arrow_type: Some(Box::new(arrow_type)), nullable: field.is_nullable(), children: Vec::new(), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), }) } } @@ -265,7 +265,7 @@ impl TryFrom<&Schema> for protobuf::Schema { fn try_from(schema: &Schema) -> Result { Ok(Self { columns: convert_arc_fields_to_proto_fields(schema.fields())?, - metadata: schema.metadata.clone(), + metadata: schema.metadata().into(), }) } } @@ -276,7 +276,7 @@ impl TryFrom for protobuf::Schema { fn try_from(schema: SchemaRef) -> Result { Ok(Self { columns: convert_arc_fields_to_proto_fields(schema.fields())?, - metadata: schema.metadata.clone(), + metadata: schema.metadata().into(), }) } } @@ -298,7 +298,7 @@ impl TryFrom<&DFSchema> for protobuf::DfSchema { .collect::, Error>>()?; Ok(Self { columns, - metadata: s.metadata().clone(), + metadata: s.metadata().into(), }) } } diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 16c3468465541..8996409f71a2f 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -400,7 +400,7 @@ pub fn serialize_expr( let expr = Box::new(protobuf::CastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), arrow_type: Some(field.data_type().try_into()?), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { @@ -411,7 +411,7 @@ pub fn serialize_expr( let expr = Box::new(protobuf::TryCastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), arrow_type: Some(field.data_type().try_into()?), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { @@ -504,7 +504,7 @@ pub fn serialize_expr( nullable: field.as_ref().map(|f| f.is_nullable()), metadata: field .as_ref() - .map(|f| f.metadata().clone()) + .map(|f| f.metadata().into()) .unwrap_or_default(), })), }, diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index a450f7a7e888f..c63ff45000c2f 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -21,8 +21,8 @@ use arrow::array::{ }; use arrow::datatypes::{ DECIMAL256_MAX_PRECISION, DataType, Field, FieldRef, Fields, Int32Type, - IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema, SchemaRef, - TimeUnit, UnionFields, UnionMode, + IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Metadata, Schema, + SchemaRef, TimeUnit, UnionFields, UnionMode, }; use arrow::util::pretty::pretty_format_batches; use datafusion::datasource::file_format::json::{JsonFormat, JsonFormatFactory}; @@ -1676,9 +1676,7 @@ async fn roundtrip_logical_plan_prepared_statement_with_metadata() -> Result<()> "".to_string(), vec![ Field::new("", DataType::Int32, true) - .with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(), ], ) diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs index 46ab8511f7002..b865940c710e0 100644 --- a/datafusion/proto/tests/cases/serialize.rs +++ b/datafusion/proto/tests/cases/serialize.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{DataType, Field, Metadata}; use datafusion::execution::FunctionRegistry; use datafusion::prelude::SessionContext; @@ -203,9 +203,7 @@ fn roundtrip_placeholder_with_metadata() { "placeholder_id".to_string(), Some( Field::new("", DataType::Utf8, false) - .with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(), ), )); diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index fd5c34ff5d961..1505070709821 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -29,7 +29,7 @@ use crate::planner::{ }; use crate::utils::normalize_ident; -use arrow::datatypes::{Field, FieldRef, Fields}; +use arrow::datatypes::{Field, FieldRef, Fields, Metadata}; use datafusion_common::error::_plan_err; use datafusion_common::format::ExplainStatementOptions; use datafusion_common::parsers::CompressionTypeVariant; @@ -2928,7 +2928,7 @@ impl SqlToRel<'_, S> { let mut planner_context = PlannerContext::new().with_prepare_param_data_types(prepare_param_data_types); planner_context.set_table_schema(Some(DFSchemaRef::new( - DFSchema::from_unqualified_fields(fields.clone(), Default::default())?, + DFSchema::from_unqualified_fields(fields.clone(), Metadata::new())?, ))); let source = self.query_to_plan(*source, &mut planner_context)?; if fields.len() != source.schema().fields().len() { diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rs index 68c560ead68cd..41eb6ef8ea4c2 100644 --- a/datafusion/sql/tests/cases/params.rs +++ b/datafusion/sql/tests/cases/params.rs @@ -16,7 +16,7 @@ // under the License. use crate::logical_plan; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use datafusion_common::{ ParamValues, ScalarValue, assert_contains, metadata::{ScalarAndMetadata, format_type_and_metadata}, @@ -732,9 +732,8 @@ fn test_prepare_statement_to_plan_one_param() { fn test_update_infer_with_metadata() { // Here the uuid field is inferred as nullable because it appears in the filter // (and not in the update values, where its nullability would be inferred) - let uuid_field = Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())].into(), - ); + let uuid_field = Field::new("", DataType::FixedSizeBinary(16), true) + .with_metadata(Metadata::new().with("ARROW:extension:name", "arrow.uuid")); let uuid_bytes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let expected_types = vec![ ( @@ -801,9 +800,8 @@ fn test_update_infer_with_metadata() { #[test] fn test_insert_infer_with_metadata() { - let uuid_field = Field::new("", DataType::FixedSizeBinary(16), false).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())].into(), - ); + let uuid_field = Field::new("", DataType::FixedSizeBinary(16), false) + .with_metadata(Metadata::new().with("ARROW:extension:name", "arrow.uuid")); let uuid_bytes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let expected_types = vec![ ("$1", Some(uuid_field.clone().with_name("id").into())), diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index e7c819bbf64a6..1a2f82506e82c 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -168,8 +168,7 @@ impl ContextProvider for MockContextProvider { ])), "person_with_uuid_extension" => Ok(Schema::new(vec![ Field::new("id", DataType::FixedSizeBinary(16), false).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), Field::new("first_name", DataType::Utf8, false), Field::new("last_name", DataType::Utf8, false), @@ -382,8 +381,7 @@ impl TypePlanner for CustomTypePlanner { match sql_type { sqlparser::ast::DataType::Uuid => Ok(Some(Arc::new( Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), ))), sqlparser::ast::DataType::Datetime(precision) => { diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index d85ca2db76268..0a551a8f69f47 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -29,7 +29,7 @@ use arrow::array::{ }; use arrow::buffer::ScalarBuffer; use arrow::datatypes::{ - DataType, Field, FieldRef, Fields, Schema, SchemaRef, TimeUnit, UInt32Type, + DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaRef, TimeUnit, UInt32Type, UnionFields, }; use arrow::record_batch::RecordBatch; @@ -82,8 +82,7 @@ impl TypePlanner for SqlLogicTestTypePlanner { match sql_type { ast::DataType::Uuid => Ok(Some(Arc::new( Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), ))), _ => Ok(None), From 6a606864aa51cf7c77388f2203e8f5b95648e485 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 10:32:13 -0400 Subject: [PATCH 3/3] Pin arrow-rs to sdf-jkl/arrow-rs alp branch to test ALP encoding (apache/arrow-rs#9372) Co-Authored-By: Claude Fable 5 --- Cargo.lock | 42 +++++++++---------- Cargo.toml | 28 ++++++------- .../common/src/file_options/parquet_writer.rs | 3 +- .../datasource-arrow/src/file_format.rs | 6 +-- datafusion/datasource-arrow/src/source.rs | 7 +--- datafusion/proto-common/src/from_proto/mod.rs | 8 +--- 6 files changed, 42 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f1e5271d82c2..6c95049fa8ac6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-arith", "arrow-array", @@ -187,7 +187,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -200,7 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "ahash", "arrow-buffer", @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -242,7 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "bytes", "half", @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "arrow-cmp" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,7 +284,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-cast", @@ -298,7 +298,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-buffer", "arrow-schema", @@ -310,7 +310,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-arith", "arrow-array", @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,7 +351,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -375,7 +375,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -388,7 +388,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -400,7 +400,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "bitflags", "serde", @@ -411,7 +411,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "ahash", "arrow-array", @@ -425,7 +425,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "arrow-array", "arrow-buffer", @@ -4563,7 +4563,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +source = "git+https://github.com/sdf-jkl/arrow-rs.git?branch=alp#5e9ad1c6fde4a0911fbaaa1244c81e18faafb804" dependencies = [ "ahash", "arrow-array", @@ -4942,7 +4942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -4961,7 +4961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", @@ -5495,7 +5495,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1bbf056f9b8f7..71ac02585e641 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -360,19 +360,19 @@ debug-assertions = false strip = "debuginfo" incremental = false -## Temporary arrow-rs patch until 60.0.0 is released +## Temporary arrow-rs patch to test apache/arrow-rs#9372 (Parquet ALP encoder/decoder support) [patch.crates-io] -arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-avro = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-data = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-select = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-string = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-ord = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } -parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-array = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-avro = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-buffer = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-cast = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-data = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-ipc = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-schema = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-select = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-string = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-ord = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +arrow-flight = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } +parquet = { git = "https://github.com/sdf-jkl/arrow-rs.git", branch = "alp" } diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs index c539245764d45..ddf18d87a9ad6 100644 --- a/datafusion/common/src/file_options/parquet_writer.rs +++ b/datafusion/common/src/file_options/parquet_writer.rs @@ -312,11 +312,12 @@ pub(crate) fn parse_encoding_string( "delta_byte_array" => Ok(parquet::basic::Encoding::DELTA_BYTE_ARRAY), "rle_dictionary" => Ok(parquet::basic::Encoding::RLE_DICTIONARY), "byte_stream_split" => Ok(parquet::basic::Encoding::BYTE_STREAM_SPLIT), + "alp" => Ok(parquet::basic::Encoding::ALP), _ => Err(DataFusionError::Configuration(format!( "Unknown or unsupported parquet encoding: \ {str_setting}. Valid values are: plain, plain_dictionary, rle, \ bit_packed, delta_binary_packed, delta_length_byte_array, \ - delta_byte_array, rle_dictionary, and byte_stream_split." + delta_byte_array, rle_dictionary, byte_stream_split, and alp." ))), } } diff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rs index c8ccc968482d1..c50ad98dfca0b 100644 --- a/datafusion/datasource-arrow/src/file_format.rs +++ b/datafusion/datasource-arrow/src/file_format.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::ArrowError; -use arrow::ipc::convert::try_fb_to_schema; +use arrow::ipc::convert::fb_to_schema; use arrow::ipc::reader::{FileReader, StreamReader}; use arrow::ipc::writer::IpcWriteOptions; use arrow::ipc::{CompressionType, root_as_message}; @@ -481,9 +481,7 @@ async fn infer_stream_schema( let fb_schema = message.header_as_schema().ok_or_else(|| { ArrowError::IpcError("Unable to read IPC message schema".to_string()) })?; - let schema = try_fb_to_schema(fb_schema).map_err(|err| { - ArrowError::IpcError(format!("Unable to convert IPC schema: {err:?}")) - })?; + let schema = fb_to_schema(fb_schema); Ok(Arc::new(schema)) } diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs index bad2ba141a68a..dba99a9758886 100644 --- a/datafusion/datasource-arrow/src/source.rs +++ b/datafusion/datasource-arrow/src/source.rs @@ -179,12 +179,7 @@ impl FileOpener for ArrowFileOpener { })?; // build decoder according to footer & projection let schema = - arrow_ipc::convert::try_fb_to_schema(footer.schema().unwrap()) - .map_err(|err| { - exec_datafusion_err!( - "Unable to convert IPC schema: {err:?}" - ) - })?; + arrow_ipc::convert::fb_to_schema(footer.schema().unwrap()); let mut decoder = FileDecoder::new(schema.into(), footer.version()); if let Some(projection) = projection { decoder = decoder.with_projection(projection); diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index df583112672aa..169ff7f3d9ff2 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -28,7 +28,7 @@ use arrow::datatypes::{ TimeUnit, UnionFields, UnionMode, i256, }; use arrow::ipc::{ - convert::try_fb_to_schema, + convert::fb_to_schema, reader::{read_dictionary, read_record_batch}, root_as_message, writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions}, @@ -461,11 +461,7 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue { .to_string(), ) })?; - try_fb_to_schema(ipc_schema).map_err(|e| { - Error::General(format!( - "Error converting IPC schema while deserializing nested ScalarValue: {e}" - )) - })? + fb_to_schema(ipc_schema) }; let message = root_as_message(ipc_message.as_slice()).map_err(|e| {