From 035eb94002ff07c14b718a8fc5bffb2098c55e88 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 09:15:04 -0400 Subject: [PATCH 1/2] Update to arrow-rs main (arrow 60 pre-release) Pin arrow/parquet crates to arrow-rs rev 901e69f913bb8a1738009218898959e8f4cd7ec8 via [patch.crates-io] until arrow 60.0.0 is released. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 225 ++++++++++++++++++++++++++++++++++++++++------------- Cargo.toml | 21 ++++- 2 files changed, 189 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7576659f0620..39b0b5ba46029 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,8 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-arith", "arrow-array", @@ -182,14 +181,13 @@ dependencies = [ "arrow-select", "arrow-string", "half", - "rand 0.9.4", + "rand 0.10.1", ] [[package]] name = "arrow-arith" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -202,8 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-buffer", @@ -222,8 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb45cd6bd2b25c0965793b83200eaca82214273a8030fbbc2d783e4c7c65a61" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -234,7 +230,7 @@ dependencies = [ "flate2", "indexmap 2.14.0", "liblzma", - "rand 0.9.4", + "rand 0.10.1", "serde", "serde_json", "snap", @@ -246,8 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "bytes", "half", @@ -258,8 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -277,11 +271,20 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cmp" +version = "59.2.0" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-csv" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-cast", @@ -295,8 +298,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-buffer", "arrow-schema", @@ -308,8 +310,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-arith", "arrow-array", @@ -335,8 +336,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,8 +351,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -376,11 +375,11 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -389,8 +388,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,8 +400,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "bitflags", "serde", @@ -414,12 +411,12 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -428,8 +425,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -522,9 +518,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" dependencies = [ "num-traits", ] @@ -1381,6 +1377,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "7.2.2" @@ -1515,6 +1521,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -3311,8 +3327,6 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.4", - "rand_distr", "zerocopy", ] @@ -3857,6 +3871,55 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4383,14 +4446,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", + "aws-lc-rs", "base64 0.22.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -4399,14 +4464,14 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5 0.11.0", + "nix", "parking_lot", "percent-encoding", "quick-xml", "rand 0.10.1", "reqwest", - "ring", "rustls-pki-types", "serde", "serde_json", @@ -4418,6 +4483,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -4498,8 +4564,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +source = "git+https://github.com/apache/arrow-rs.git?rev=901e69f913bb8a1738009218898959e8f4cd7ec8#901e69f913bb8a1738009218898959e8f4cd7ec8" dependencies = [ "ahash", "arrow-array", @@ -4933,9 +4998,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -4967,6 +5032,7 @@ version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -5255,9 +5321,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64 0.22.1", "bytes", @@ -5276,11 +5342,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", + "rustls-platform-verifier", "sync_wrapper", "tokio", "tokio-rustls", @@ -5415,6 +5478,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -5785,6 +5875,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -5849,6 +5949,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqllogictest" version = "0.29.1" @@ -6989,9 +7095,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -7032,6 +7138,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "whoami" version = "2.1.1" diff --git a/Cargo.toml b/Cargo.toml index 65670d895e1a8..1bbf056f9b8f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,7 +103,7 @@ arrow-avro = { version = "59.2.0", default-features = false, features = [ arrow-buffer = { version = "59.2.0", default-features = false } arrow-data = { version = "59.2.0", default-features = false } arrow-flight = { version = "59.2.0", features = [ - "flight-sql-experimental", + "flight-sql", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. @@ -176,7 +176,7 @@ liblzma = { version = "0.4.6", features = ["static"] } log = "^0.4" memchr = "2.8.1" num-traits = { version = "0.2" } -object_store = { version = "0.13.2", default-features = false } +object_store = { version = "0.14.1", default-features = false } parking_lot = "0.12" parquet = { version = "59.2.0", default-features = false, features = [ "arrow", @@ -359,3 +359,20 @@ debug = false debug-assertions = false strip = "debuginfo" incremental = false + +## Temporary arrow-rs patch until 60.0.0 is released + +[patch.crates-io] +arrow = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-avro = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-cast = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-data = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-select = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-string = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-ord = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } +parquet = { git = "https://github.com/apache/arrow-rs.git", rev = "901e69f913bb8a1738009218898959e8f4cd7ec8" } From 70bd16a16c16126b70ab9426e1ecf98faf8e91a0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 10:19:41 -0400 Subject: [PATCH 2/2] Fix compilation for arrow-rs main (Metadata type, rand 0.10, object_store 0.14, etc) Co-Authored-By: Claude Fable 5 --- datafusion-cli/src/main.rs | 20 ++-- datafusion/catalog-listing/src/helpers.rs | 4 +- datafusion/catalog-listing/src/table.rs | 4 +- datafusion/common/src/dfschema.rs | 17 ++-- datafusion/common/src/heap_size.rs | 15 ++- datafusion/common/src/metadata.rs | 94 ++++++++++--------- datafusion/common/src/nested_struct.rs | 12 +-- datafusion/common/src/param_value.rs | 2 +- .../core/src/datasource/file_format/csv.rs | 1 + datafusion/core/tests/sql/path_partition.rs | 2 + .../user_defined/user_defined_aggregates.rs | 16 +--- .../user_defined_scalar_functions.rs | 9 +- .../user_defined_window_functions.rs | 9 +- .../datasource-arrow/src/file_format.rs | 6 +- datafusion/datasource-arrow/src/source.rs | 7 +- .../datasource-parquet/src/file_format.rs | 4 +- datafusion/execution/src/cache/mod.rs | 6 +- datafusion/expr/src/expr.rs | 66 ++++++------- datafusion/expr/src/logical_plan/builder.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 7 +- datafusion/ffi/src/record_batch_stream.rs | 4 +- datafusion/ffi/src/udaf/mod.rs | 8 +- .../functions-aggregate/benches/array_agg.rs | 4 +- .../functions-aggregate/src/array_agg.rs | 4 +- datafusion/functions/src/core/arrow_field.rs | 17 +++- .../functions/src/core/arrow_metadata.rs | 16 +++- .../functions/src/core/with_metadata.rs | 11 +-- datafusion/optimizer/src/optimizer.rs | 5 +- .../optimizer/src/propagate_empty_relation.rs | 4 +- datafusion/optimizer/src/push_down_filter.rs | 4 +- .../src/simplify_expressions/udf_preimage.rs | 6 +- datafusion/physical-expr/benches/case_when.rs | 9 +- datafusion/physical-expr/src/planner.rs | 9 +- datafusion/physical-plan/src/joins/utils.rs | 8 +- .../physical-plan/src/spill/spill_manager.rs | 33 ++----- datafusion/physical-plan/src/union.rs | 4 +- datafusion/proto-common/src/from_proto/mod.rs | 8 +- datafusion/proto-common/src/to_proto/mod.rs | 8 +- datafusion/proto/src/logical_plan/to_proto.rs | 6 +- .../tests/cases/roundtrip_logical_plan.rs | 8 +- datafusion/proto/tests/cases/serialize.rs | 6 +- datafusion/sql/src/statement.rs | 4 +- datafusion/sql/tests/cases/params.rs | 12 +-- datafusion/sql/tests/common/mod.rs | 6 +- datafusion/sqllogictest/src/test_context.rs | 5 +- .../datetime/arith_interval_double.slt | 10 +- .../test_files/push_down_filter_parquet.slt | 44 ++++----- 47 files changed, 289 insertions(+), 283 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 4be6058aef8b6..ba40587ee5812 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -662,9 +662,9 @@ mod tests { +-----------------------------------+-----------------+---------------------+------+------------------+ | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ - | alltypes_plain.parquet | 1851 | 8794 | 1 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 1 | page_index=false | + | alltypes_plain.parquet | 1851 | 8970 | 1 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 269562 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 1 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); @@ -693,9 +693,9 @@ mod tests { +-----------------------------------+-----------------+---------------------+------+------------------+ | filename | file_size_bytes | metadata_size_bytes | hits | extra | +-----------------------------------+-----------------+---------------------+------+------------------+ - | alltypes_plain.parquet | 1851 | 8794 | 4 | page_index=false | - | alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true | - | lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false | + | alltypes_plain.parquet | 1851 | 8970 | 4 | page_index=false | + | alltypes_tiny_pages.parquet | 454233 | 269562 | 2 | page_index=true | + | lz4_raw_compressed_larger.parquet | 380836 | 1347 | 2 | page_index=false | +-----------------------------------+-----------------+---------------------+------+------------------+ "); @@ -862,14 +862,14 @@ mod tests { ])? .sort(vec![col("filename").sort(true, false)])?; let rbs = df.collect().await?; - assert_snapshot!(batches_to_string(&rbs),@r" + assert_snapshot!(batches_to_string(&rbs),@r#" +---------------------+-----------+-----------------+------+------+ | metadata_size_bytes | filename | file_size_bytes | etag | hits | +---------------------+-----------+-----------------+------+------+ - | 212 | 0.parquet | 3642 | 0 | 2 | - | 212 | 1.parquet | 3642 | 1 | 2 | + | 216 | 0.parquet | 3620 | "0" | 2 | + | 216 | 1.parquet | 3620 | "1" | 2 | +---------------------+-----------+-----------------+------+------+ - "); + "#); Ok(()) } diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index 098f3d51ef911..b7cf6af3b7faa 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -30,7 +30,7 @@ use datafusion_expr::{BinaryExpr, Operator, lit, utils}; use arrow::{ array::AsArray, - datatypes::{DataType, Field}, + datatypes::{DataType, Field, Metadata}, record_batch::RecordBatch, }; use datafusion_expr::execution_props::ExecutionProps; @@ -423,7 +423,7 @@ pub async fn pruned_partition_list<'a>( .iter() .map(|(n, d)| Field::new(n, d.clone(), true)) .collect(), - Default::default(), + Metadata::new(), )?; Ok(objects diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index 498640ba0ff32..8ff92accdf642 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -20,7 +20,7 @@ use crate::helpers::{ expr_applicable_for_cols, filter_partitioned_file, pruned_partition_list, }; use crate::{ListingOptions, ListingTableConfig}; -use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef}; +use arrow::datatypes::{Field, Metadata, Schema, SchemaBuilder, SchemaRef}; use async_trait::async_trait; use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider}; use datafusion_common::stats::Precision; @@ -1041,7 +1041,7 @@ impl ListingTable { .iter() .map(|(name, data_type)| Field::new(name, data_type.clone(), true)) .collect(), - Default::default(), + Metadata::new(), )?; file_groups diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 262f1dcf619d9..a49879e466b26 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -18,7 +18,7 @@ //! DFSchema is an extended schema struct that DataFusion uses to provide support for //! fields with optional relation names. -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashSet}; use std::fmt::{Display, Formatter}; use std::hash::Hash; use std::sync::{Arc, LazyLock}; @@ -31,7 +31,7 @@ use crate::{ use arrow::compute::can_cast_types; use arrow::datatypes::{ - DataType, Field, FieldRef, Fields, Schema, SchemaBuilder, SchemaRef, + DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaBuilder, SchemaRef, }; /// A reference-counted reference to a [DFSchema]. @@ -153,7 +153,7 @@ impl DFSchema { /// Create a `DFSchema` from an Arrow schema where all the fields have a given qualifier pub fn new_with_metadata( qualified_fields: Vec<(Option, Arc)>, - metadata: HashMap, + metadata: impl Into, ) -> Result { let (qualifiers, fields): (Vec>, Vec>) = qualified_fields.into_iter().unzip(); @@ -172,7 +172,7 @@ impl DFSchema { /// Create a new `DFSchema` from a list of Arrow [Field]s pub fn from_unqualified_fields( fields: Fields, - metadata: HashMap, + metadata: impl Into, ) -> Result { let field_count = fields.len(); let schema = Arc::new(Schema::new_with_metadata(fields, metadata)); @@ -864,7 +864,7 @@ impl DFSchema { } /// Get metadata of this schema - pub fn metadata(&self) -> &HashMap { + pub fn metadata(&self) -> &Metadata { &self.inner.metadata } @@ -1179,7 +1179,7 @@ impl ToDFSchema for Vec { let field_count = self.len(); let schema = Schema { fields: self.into(), - metadata: HashMap::new(), + metadata: Metadata::new(), }; let dfschema = DFSchema { inner: schema.into(), @@ -1221,7 +1221,7 @@ pub trait ExprSchema: std::fmt::Debug { } /// Returns the column's optional metadata. - fn metadata(&self, col: &Column) -> Result<&HashMap> { + fn metadata(&self, col: &Column) -> Result<&Metadata> { Ok(self.field_from_column(col)?.metadata()) } @@ -1245,7 +1245,7 @@ impl + std::fmt::Debug> ExprSchema for P { self.as_ref().data_type(col) } - fn metadata(&self, col: &Column) -> Result<&HashMap> { + fn metadata(&self, col: &Column) -> Result<&Metadata> { ExprSchema::metadata(self.as_ref(), col) } @@ -1379,6 +1379,7 @@ pub fn qualified_name(qualifier: Option<&TableReference>, name: &str) -> String #[cfg(test)] mod tests { use crate::assert_contains; + use std::collections::HashMap; use super::*; diff --git a/datafusion/common/src/heap_size.rs b/datafusion/common/src/heap_size.rs index 037f807fce9d8..7f774fd5a6826 100644 --- a/datafusion/common/src/heap_size.rs +++ b/datafusion/common/src/heap_size.rs @@ -47,7 +47,7 @@ use arrow::array::{ }; use arrow::datatypes::{ DataType, Field, Fields, IntervalDayTime, IntervalMonthDayNano, IntervalUnit, - TimeUnit, UnionFields, UnionMode, i256, + Metadata, TimeUnit, UnionFields, UnionMode, i256, }; use chrono::{DateTime, Utc}; use half::f16; @@ -396,6 +396,19 @@ impl DFHeapSize for UnionFields { } } +impl DFHeapSize for Metadata { + fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize { + // `Metadata` does not expose its underlying reference-counted map, so + // this approximates the `BTreeMap` entries' sizes and cannot dedupe + // instances that share the same allocation. + self.iter() + .map(|(k, v)| { + size_of::<(String, String)>() + k.heap_size(ctx) + v.heap_size(ctx) + }) + .sum() + } +} + impl DFHeapSize for Field { fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize { self.name().heap_size(ctx) diff --git a/datafusion/common/src/metadata.rs b/datafusion/common/src/metadata.rs index d6d8fb7b0ed0c..34242e9b52757 100644 --- a/datafusion/common/src/metadata.rs +++ b/datafusion/common/src/metadata.rs @@ -17,7 +17,7 @@ use std::{collections::BTreeMap, sync::Arc}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use hashbrown::HashMap; use crate::{DataFusionError, ScalarValue, error::_plan_err}; @@ -84,14 +84,8 @@ impl From for ScalarAndMetadata { /// Returns a planning error with suitably formatted type representations if /// actual and expected do not compare to equal. pub fn check_metadata_with_storage_equal( - actual: ( - &DataType, - Option<&std::collections::HashMap>, - ), - expected: ( - &DataType, - Option<&std::collections::HashMap>, - ), + actual: (&DataType, Option<&Metadata>), + expected: (&DataType, Option<&Metadata>), what: &str, context: &str, ) -> Result<(), DataFusionError> { @@ -131,7 +125,7 @@ pub fn check_metadata_with_storage_equal( /// renderings. pub fn format_type_and_metadata( data_type: &DataType, - metadata: Option<&std::collections::HashMap>, + metadata: Option<&Metadata>, ) -> String { match metadata { Some(metadata) if !metadata.is_empty() => { @@ -179,10 +173,7 @@ pub fn format_type_and_metadata( pub struct FieldMetadata { /// The inner metadata of a literal expression, which is a map of string /// keys to string values. - /// - /// Note this is not a `HashMap` because `HashMap` does not provide - /// implementations for traits like `Debug` and `Hash`. - inner: Arc>, + inner: Metadata, } impl Default for FieldMetadata { @@ -195,7 +186,7 @@ impl FieldMetadata { /// Create a new empty metadata instance. pub fn new_empty() -> Self { Self { - inner: Arc::new(BTreeMap::new()), + inner: Metadata::new(), } } @@ -261,31 +252,28 @@ impl FieldMetadata { } /// Create a new metadata instance from a `Field`'s metadata. + /// + /// This is cheap, as [`Metadata`] is cheap to clone. pub fn new_from_field(field: &Field) -> Self { - let inner = field - .metadata() - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); Self { - inner: Arc::new(inner), + inner: field.metadata().clone(), } } /// Create a new metadata instance from a map of string keys to string values. pub fn new(inner: BTreeMap) -> Self { Self { - inner: Arc::new(inner), + inner: Metadata::from(inner), } } - /// Get the inner metadata as a reference to a `BTreeMap`. - pub fn inner(&self) -> &BTreeMap { + /// Get the inner [`Metadata`] + pub fn inner(&self) -> &Metadata { &self.inner } - /// Return the inner metadata - pub fn into_inner(self) -> Arc> { + /// Return the inner [`Metadata`] + pub fn into_inner(self) -> Metadata { self.inner } @@ -294,8 +282,7 @@ impl FieldMetadata { if other.is_empty() { return; } - let other = Arc::unwrap_or_clone(other.into_inner()); - Arc::make_mut(&mut self.inner).extend(other); + self.inner.extend(other.inner); } /// Returns true if the metadata is empty. @@ -310,10 +297,14 @@ impl FieldMetadata { /// Convert this `FieldMetadata` into a `HashMap` pub fn to_hashmap(&self) -> std::collections::HashMap { - self.inner - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect() + (&self.inner).into() + } + + /// Convert this `FieldMetadata` into an arrow [`Metadata`] + /// + /// This is cheap, as [`Metadata`] is cheap to clone. + pub fn to_metadata(&self) -> Metadata { + self.inner.clone() } /// Updates the metadata on the Field with this metadata, if it is not empty. @@ -322,7 +313,7 @@ impl FieldMetadata { return field; } - field.with_metadata(self.to_hashmap()) + field.with_metadata(self.inner.clone()) } /// Updates the metadata on the FieldRef with this metadata, if it is not empty. @@ -331,11 +322,29 @@ impl FieldMetadata { return field_ref; } - Arc::make_mut(&mut field_ref).set_metadata(self.to_hashmap()); + Arc::make_mut(&mut field_ref).set_metadata(self.inner.clone()); field_ref } } +impl From<&FieldMetadata> for Metadata { + fn from(value: &FieldMetadata) -> Self { + value.to_metadata() + } +} + +impl From for FieldMetadata { + fn from(value: Metadata) -> Self { + Self { inner: value } + } +} + +impl From<&Metadata> for FieldMetadata { + fn from(value: &Metadata) -> Self { + Self::from(value.clone()) + } +} + impl From<&Field> for FieldMetadata { fn from(field: &Field) -> Self { Self::new_from_field(field) @@ -350,7 +359,9 @@ impl From> for FieldMetadata { impl From> for FieldMetadata { fn from(map: std::collections::HashMap) -> Self { - Self::new(map.into_iter().collect()) + Self { + inner: Metadata::from(map), + } } } @@ -368,17 +379,16 @@ impl From<&std::collections::HashMap> for FieldMetadata { /// From hashbrown map impl From> for FieldMetadata { fn from(map: HashMap) -> Self { - let inner = map.into_iter().collect(); - Self::new(inner) + Self { + inner: map.into_iter().collect(), + } } } impl From<&HashMap> for FieldMetadata { fn from(map: &HashMap) -> Self { - let inner = map - .into_iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - Self::new(inner) + Self { + inner: map.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + } } } diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index e915b91b911cc..da994f7a95196 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -1123,8 +1123,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1148,8 +1148,8 @@ mod tests { Arc::new(non_null_field( "entries", struct_type(vec![ - non_null_field("keys", DataType::Utf8), - field("values", DataType::Int32), + non_null_field("key", DataType::Utf8), + field("value", DataType::Int32), ]), )), false, @@ -1176,8 +1176,8 @@ mod tests { assert!(map.is_null(1)); let map0 = map.value(0); let entries = map0.as_any().downcast_ref::().unwrap(); - let keys = get_column_as!(entries, "keys", StringArray); - let vals = get_column_as!(entries, "values", Int32Array); + let keys = get_column_as!(entries, "key", StringArray); + let vals = get_column_as!(entries, "value", Int32Array); assert_eq!(keys.value(0), "a"); assert_eq!(vals.value(0), 1); } diff --git a/datafusion/common/src/param_value.rs b/datafusion/common/src/param_value.rs index 0fac6b529eb0f..1979daa7e3755 100644 --- a/datafusion/common/src/param_value.rs +++ b/datafusion/common/src/param_value.rs @@ -64,7 +64,7 @@ impl ParamValues { check_metadata_with_storage_equal( ( &lit.value.data_type(), - lit.metadata.as_ref().map(|m| m.to_hashmap()).as_ref(), + lit.metadata.as_ref().map(|m| m.to_metadata()).as_ref(), ), (param_type.data_type(), Some(param_type.metadata())), "parameter", diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 90d7eb3b41388..b9cc06385996b 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -136,6 +136,7 @@ mod tests { }, range: Default::default(), attributes: Attributes::default(), + extensions: Default::default(), }) } diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs index 82a15eb401fc4..4fce490d3fce7 100644 --- a/datafusion/core/tests/sql/path_partition.rs +++ b/datafusion/core/tests/sql/path_partition.rs @@ -718,6 +718,7 @@ impl ObjectStore for MirroringObjectStore { payload, meta, attributes: Attributes::default(), + extensions: Default::default(), }) } @@ -789,6 +790,7 @@ impl ObjectStore for MirroringObjectStore { Ok(ListResult { common_prefixes: common_prefixes.into_iter().collect(), objects, + extensions: Default::default(), }) } diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs index 323925bcfaf82..12e72f6091c8a 100644 --- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs +++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs @@ -30,7 +30,7 @@ use arrow::array::{ Array, AsArray, Int32Array, PrimitiveArray, StringArray, StructArray, UInt64Array, record_batch, types::UInt64Type, }; -use arrow::datatypes::{Fields, Schema}; +use arrow::datatypes::{Fields, Metadata, Schema}; use arrow_schema::FieldRef; use datafusion::common::test_util::batches_to_string; use datafusion::dataframe::DataFrame; @@ -1018,11 +1018,8 @@ async fn test_metadata_based_aggregate() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( @@ -1093,11 +1090,8 @@ async fn test_metadata_based_aggregate_as_window() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index b758aeb5209e8..474b86fb65aeb 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -25,7 +25,7 @@ use arrow::array::{ }; use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch}; use arrow::compute::kernels::numeric::add; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Metadata, Schema}; use arrow_schema::extension::{Bool8, CanonicalExtensionType, ExtensionType}; use arrow_schema::{ArrowError, FieldRef, SchemaRef}; use datafusion::common::test_util::batches_to_string; @@ -1645,11 +1645,8 @@ async fn test_metadata_based_udf() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( schema, diff --git a/datafusion/core/tests/user_defined/user_defined_window_functions.rs b/datafusion/core/tests/user_defined/user_defined_window_functions.rs index afaf269ca1200..ea789bb9a4c39 100644 --- a/datafusion/core/tests/user_defined/user_defined_window_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_window_functions.rs @@ -22,7 +22,7 @@ use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, RecordBatch, StringArray, UInt64Array, record_batch, }; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Metadata, Schema}; use arrow_schema::FieldRef; use datafusion::common::test_util::batches_to_string; use datafusion::common::{Result, ScalarValue}; @@ -868,11 +868,8 @@ async fn test_metadata_based_window_fn() -> Result<()> { let data_array = Arc::new(UInt64Array::from(vec![0, 5, 10, 15, 20])) as ArrayRef; let schema = Arc::new(Schema::new(vec![ Field::new("no_metadata", DataType::UInt64, true), - Field::new("with_metadata", DataType::UInt64, true).with_metadata( - [("modify_values".to_string(), "double_output".to_string())] - .into_iter() - .collect(), - ), + Field::new("with_metadata", DataType::UInt64, true) + .with_metadata(Metadata::new().with("modify_values", "double_output")), ])); let batch = RecordBatch::try_new( diff --git a/datafusion/datasource-arrow/src/file_format.rs b/datafusion/datasource-arrow/src/file_format.rs index c50ad98dfca0b..c8ccc968482d1 100644 --- a/datafusion/datasource-arrow/src/file_format.rs +++ b/datafusion/datasource-arrow/src/file_format.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use arrow::datatypes::{Schema, SchemaRef}; use arrow::error::ArrowError; -use arrow::ipc::convert::fb_to_schema; +use arrow::ipc::convert::try_fb_to_schema; use arrow::ipc::reader::{FileReader, StreamReader}; use arrow::ipc::writer::IpcWriteOptions; use arrow::ipc::{CompressionType, root_as_message}; @@ -481,7 +481,9 @@ async fn infer_stream_schema( let fb_schema = message.header_as_schema().ok_or_else(|| { ArrowError::IpcError("Unable to read IPC message schema".to_string()) })?; - let schema = fb_to_schema(fb_schema); + let schema = try_fb_to_schema(fb_schema).map_err(|err| { + ArrowError::IpcError(format!("Unable to convert IPC schema: {err:?}")) + })?; Ok(Arc::new(schema)) } diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs index dba99a9758886..bad2ba141a68a 100644 --- a/datafusion/datasource-arrow/src/source.rs +++ b/datafusion/datasource-arrow/src/source.rs @@ -179,7 +179,12 @@ impl FileOpener for ArrowFileOpener { })?; // build decoder according to footer & projection let schema = - arrow_ipc::convert::fb_to_schema(footer.schema().unwrap()); + arrow_ipc::convert::try_fb_to_schema(footer.schema().unwrap()) + .map_err(|err| { + exec_datafusion_err!( + "Unable to convert IPC schema: {err:?}" + ) + })?; let mut decoder = FileDecoder::new(schema.into(), footer.version()); if let Some(projection) = projection { decoder = decoder.with_projection(projection); diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs index 6358201c06fa5..bf35ad5a8c663 100644 --- a/datafusion/datasource-parquet/src/file_format.rs +++ b/datafusion/datasource-parquet/src/file_format.rs @@ -32,7 +32,7 @@ pub use crate::schema_coercion::{ pub use crate::sink::ParquetSink; -use arrow::datatypes::{Fields, Schema, SchemaRef}; +use arrow::datatypes::{Fields, Metadata, Schema, SchemaRef}; use datafusion_datasource::TableSchema; use datafusion_datasource::file_compression_type::FileCompressionType; use datafusion_datasource::file_sink_config::FileSinkConfig; @@ -266,7 +266,7 @@ fn clear_metadata( .fields() .iter() .map(|field| { - field.as_ref().clone().with_metadata(Default::default()) // clear meta + field.as_ref().clone().with_metadata(Metadata::new()) // clear meta }) .collect::(); Schema::new(fields) diff --git a/datafusion/execution/src/cache/mod.rs b/datafusion/execution/src/cache/mod.rs index f47a3f3ca49f3..2eba533e04b67 100644 --- a/datafusion/execution/src/cache/mod.rs +++ b/datafusion/execution/src/cache/mod.rs @@ -228,7 +228,7 @@ impl DFHeapSize for SchemaFingerprint { #[cfg(test)] mod schema_fingerprint_tests { use super::*; - use datafusion_common::arrow::datatypes::Field; + use datafusion_common::arrow::datatypes::{Field, Metadata}; fn fp(fields: Vec) -> SchemaFingerprint { SchemaFingerprint::from_schema(&Schema::new(fields)) @@ -263,13 +263,13 @@ mod schema_fingerprint_tests { let field_md = SchemaFingerprint::from_schema(&Schema::new(vec![ Field::new("id", DataType::Int64, false) - .with_metadata([("note".to_string(), "x".to_string())].into()), + .with_metadata(Metadata::new().with("note", "x")), ])); assert_eq!(plain, field_md, "field metadata must be ignored"); let schema_md = SchemaFingerprint::from_schema( &Schema::new(vec![Field::new("id", DataType::Int64, false)]) - .with_metadata([("k".to_string(), "v".to_string())].into()), + .with_metadata(Metadata::new().with("k", "v")), ); assert_eq!(plain, schema_md, "schema metadata must be ignored"); } diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index f9c0662e682e8..82186f569be85 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -32,7 +32,7 @@ use crate::type_coercion::functions::value_fields_with_higher_order_udf; use crate::{AggregateUDF, LambdaParametersProgress, ValueOrLambda, Volatility}; use crate::{ExprSchemable, Operator, Signature, WindowFrame, WindowUDF}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use datafusion_common::cse::{HashNode, NormalizeEq, Normalizeable}; use datafusion_common::datatype::DataTypeExt; use datafusion_common::metadata::format_type_and_metadata; @@ -622,7 +622,7 @@ impl<'a> TreeNodeContainer<'a, Self> for Expr { /// See the [default_column_values.rs] example implementation. /// /// [default_column_values.rs]: https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/custom_data_source/default_column_values.rs -pub type SchemaFieldMetadata = std::collections::HashMap; +pub type SchemaFieldMetadata = Metadata; /// Intersects multiple metadata instances for UNION operations. /// @@ -648,7 +648,7 @@ pub type SchemaFieldMetadata = std::collections::HashMap; pub fn intersect_metadata_for_union<'a>( metadatas: impl IntoIterator, ) -> SchemaFieldMetadata { - let mut intersected: Option = None; + let mut intersected: Option> = None; for metadata in metadatas { // Skip empty metadata (e.g. from NULL literals or computed expressions) @@ -658,7 +658,12 @@ pub fn intersect_metadata_for_union<'a>( } match &mut intersected { None => { - intersected = Some(metadata.clone()); + intersected = Some( + metadata + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ); } Some(current) => { // Only keep keys that exist in both with the same value @@ -667,7 +672,7 @@ pub fn intersect_metadata_for_union<'a>( } } - intersected.unwrap_or_default() + intersected.map(Metadata::from).unwrap_or_default() } /// UNNEST expression. @@ -3901,7 +3906,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -3949,7 +3954,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4003,7 +4008,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4056,7 +4061,7 @@ mod test { let subquery_schema = Arc::new( DFSchema::from_unqualified_fields( vec![subquery_field].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ); @@ -4157,9 +4162,8 @@ mod test { fn infer_placeholder_with_metadata() { // name == $1, where name is a non-nullable string let schema = Arc::new(Schema::new(vec![ - Field::new("name", DataType::Utf8, false).with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ), + Field::new("name", DataType::Utf8, false) + .with_metadata(Metadata::new().with("some_key", "some_value")), ])); let df_schema = DFSchema::try_from(schema).unwrap(); @@ -4593,53 +4597,53 @@ mod test { mod intersect_metadata_tests { use super::super::intersect_metadata_for_union; - use std::collections::HashMap; + use arrow::datatypes::Metadata; #[test] fn all_branches_same_metadata() { - let m1 = HashMap::from([("key".into(), "val".into())]); - let m2 = HashMap::from([("key".into(), "val".into())]); + let m1 = Metadata::new().with("key", "val"); + let m2 = Metadata::new().with("key", "val"); let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn conflicting_metadata_dropped() { - let m1 = HashMap::from([("key".into(), "a".into())]); - let m2 = HashMap::from([("key".into(), "b".into())]); + let m1 = Metadata::new().with("key", "a"); + let m2 = Metadata::new().with("key", "b"); let result = intersect_metadata_for_union([&m1, &m2]); assert!(result.is_empty()); } #[test] fn empty_metadata_branch_skipped() { - let m1 = HashMap::from([("key".into(), "val".into())]); - let m2 = HashMap::new(); // e.g. NULL literal + let m1 = Metadata::new().with("key", "val"); + let m2 = Metadata::new(); // e.g. NULL literal let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn empty_metadata_first_branch_skipped() { - let m1 = HashMap::new(); - let m2 = HashMap::from([("key".into(), "val".into())]); + let m1 = Metadata::new(); + let m2 = Metadata::new().with("key", "val"); let result = intersect_metadata_for_union([&m1, &m2]); - assert_eq!(result, HashMap::from([("key".into(), "val".into())])); + assert_eq!(result, Metadata::new().with("key", "val")); } #[test] fn all_branches_empty_metadata() { - let m1: HashMap = HashMap::new(); - let m2: HashMap = HashMap::new(); + let m1 = Metadata::new(); + let m2 = Metadata::new(); let result = intersect_metadata_for_union([&m1, &m2]); assert!(result.is_empty()); } #[test] fn mixed_empty_and_conflicting() { - let m1 = HashMap::from([("key".into(), "a".into())]); - let m2 = HashMap::new(); - let m3 = HashMap::from([("key".into(), "b".into())]); + let m1 = Metadata::new().with("key", "a"); + let m2 = Metadata::new(); + let m3 = Metadata::new().with("key", "b"); let result = intersect_metadata_for_union([&m1, &m2, &m3]); // m2 is skipped; m1 and m3 conflict → dropped assert!(result.is_empty()); @@ -4647,9 +4651,7 @@ mod test { #[test] fn no_inputs() { - let result = intersect_metadata_for_union(std::iter::empty::< - &HashMap, - >()); + let result = intersect_metadata_for_union(std::iter::empty::<&Metadata>()); assert!(result.is_empty()); } } diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 4b728fe4474bc..48fea79493bf6 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -1765,12 +1765,8 @@ pub fn build_join_schema( _ => (right, left), }; - let metadata = schema1 - .metadata() - .clone() - .into_iter() - .chain(schema2.metadata().clone()) - .collect(); + let mut metadata = schema1.metadata().clone(); + metadata.extend(schema2.metadata().clone()); let dfschema = DFSchema::new_with_metadata(qualified_fields, metadata)?; dfschema.with_functional_dependencies(func_dependencies) diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 1a141ea52a13a..129be81faa533 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -53,7 +53,7 @@ use crate::{ use crate::statistics::StatisticsRequest; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef}; use datafusion_common::cse::{NormalizeEq, Normalizeable}; use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType}; use datafusion_common::metadata::check_metadata_with_storage_equal; @@ -3269,8 +3269,7 @@ impl Union { inputs: &[Arc], loose_types: bool, ) -> Result { - type FieldData<'a> = - (&'a DataType, bool, Vec<&'a HashMap>, usize); + type FieldData<'a> = (&'a DataType, bool, Vec<&'a Metadata>, usize); let mut cols: Vec<(&str, FieldData)> = Vec::new(); for input in inputs.iter() { for field in input.schema().fields() { @@ -5658,7 +5657,7 @@ mod tests { let schema_with_metadata = || { DFSchema::from_unqualified_fields( vec![Field::new("count", DataType::Int64, false)].into(), - [("key".to_string(), "value".to_string())].into(), + Metadata::new().with("key", "value"), ) .unwrap() }; diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs index 5a92cbfe5fe78..2866460c1825c 100644 --- a/datafusion/ffi/src/record_batch_stream.rs +++ b/datafusion/ffi/src/record_batch_stream.rs @@ -219,7 +219,7 @@ mod tests { use std::sync::Arc; use arrow::array::record_batch; - use arrow::datatypes::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Metadata, Schema}; use datafusion::error::Result; use datafusion::execution::SendableRecordBatchStream; use datafusion::test_util::bounded_stream; @@ -274,7 +274,7 @@ mod tests { .schema() .as_ref() .clone() - .with_metadata([("some_key".to_owned(), "some_value".to_owned())].into()) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(); let rb = rb.with_schema(schema)?; diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs index b3a087e5d0022..0fa15d5aacf39 100644 --- a/datafusion/ffi/src/udaf/mod.rs +++ b/datafusion/ffi/src/udaf/mod.rs @@ -657,9 +657,8 @@ impl From for FFI_AggregateOrderSensitivity { #[cfg(test)] mod tests { - use std::collections::HashMap; - use arrow::datatypes::Schema; + use arrow::datatypes::{Metadata, Schema}; use datafusion::common::create_array; use datafusion::functions_aggregate::sum::Sum; use datafusion::physical_expr::PhysicalSortExpr; @@ -776,10 +775,7 @@ mod tests { let foreign_udaf: Arc = (&local_udaf).into(); let foreign_udaf = AggregateUDF::new_from_shared_impl(foreign_udaf); - let metadata: HashMap = - [("a_key".to_string(), "a_value".to_string())] - .into_iter() - .collect(); + let metadata = Metadata::new().with("a_key", "a_value"); let input_field = Arc::new( Field::new("a", DataType::Float64, false).with_metadata(metadata.clone()), ); diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index d7e5a511078a5..4d88df4200d7c 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -33,7 +33,6 @@ use arrow::buffer::OffsetBuffer; use arrow::util::bench_util::create_primitive_array; use rand::Rng; use rand::SeedableRng; -use rand::distr::{Distribution, StandardUniform}; use rand::prelude::StdRng; /// Returns fixed seedable RNG @@ -65,8 +64,7 @@ pub fn create_list_array( zero_length_lists_probability: f32, ) -> ListArray where - T: ArrowPrimitiveType, - StandardUniform: Distribution, + T: ArrowPrimitiveType, { let mut nulls_builder = NullBufferBuilder::new(size); let mut rng = StdRng::seed_from_u64(42); diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index 0e02ff118678f..3fe8e44220d30 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -1760,7 +1760,7 @@ mod tests { acc2.update_batch(&[string_list_data([vec!["e", "f", "g"]])])?; acc1 = merge(acc1, acc2)?; - assert_eq!(acc1.size(), 2274); + assert_eq!(acc1.size(), 2194); Ok(()) } @@ -1780,7 +1780,7 @@ mod tests { ])])?; // without compaction, the size is 17112 - assert_eq!(acc.size(), 2224); + assert_eq!(acc.size(), 2144); Ok(()) } diff --git a/datafusion/functions/src/core/arrow_field.rs b/datafusion/functions/src/core/arrow_field.rs index dce7cff42ba80..de8415dd55ce0 100644 --- a/datafusion/functions/src/core/arrow_field.rs +++ b/datafusion/functions/src/core/arrow_field.rs @@ -16,7 +16,8 @@ // under the License. use arrow::array::{ - Array, BooleanArray, MapBuilder, StringArray, StringBuilder, StructArray, + Array, BooleanArray, MapBuilder, MapFieldNames, StringArray, StringBuilder, + StructArray, }; use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::{Result, ScalarValue, utils::take_function_args}; @@ -125,8 +126,18 @@ impl ScalarUDFImpl for ArrowFieldFunc { // Build the metadata map array (same pattern as arrow_metadata.rs) let metadata = field.metadata(); - let mut map_builder = - MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + // Match the field names declared in `return_type` (the arrow-rs + // default changed to `key`/`value` in arrow 60) + let map_field_names = MapFieldNames { + entry: "entries".to_string(), + key: "keys".to_string(), + value: "values".to_string(), + }; + let mut map_builder = MapBuilder::new( + Some(map_field_names), + StringBuilder::new(), + StringBuilder::new(), + ); let mut entries: Vec<_> = metadata.iter().collect(); entries.sort_by_key(|(k, _)| *k); diff --git a/datafusion/functions/src/core/arrow_metadata.rs b/datafusion/functions/src/core/arrow_metadata.rs index a80f66f396731..e0bec18f16511 100644 --- a/datafusion/functions/src/core/arrow_metadata.rs +++ b/datafusion/functions/src/core/arrow_metadata.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{MapBuilder, StringBuilder}; +use arrow::array::{MapBuilder, MapFieldNames, StringBuilder}; use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::types::logical_string; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; @@ -131,8 +131,18 @@ impl ScalarUDFImpl for ArrowMetadataFunc { let value = metadata.get(key).cloned(); Ok(ColumnarValue::Scalar(ScalarValue::Utf8(value))) } else if args.args.len() == 1 { - let mut map_builder = - MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + // Match the field names declared in `return_type` (the arrow-rs + // default changed to `key`/`value` in arrow 60) + let map_field_names = MapFieldNames { + entry: "entries".to_string(), + key: "keys".to_string(), + value: "values".to_string(), + }; + let mut map_builder = MapBuilder::new( + Some(map_field_names), + StringBuilder::new(), + StringBuilder::new(), + ); let mut entries: Vec<_> = metadata.iter().collect(); entries.sort_by_key(|(k, _)| *k); diff --git a/datafusion/functions/src/core/with_metadata.rs b/datafusion/functions/src/core/with_metadata.rs index 481ed713ed7ad..189f6360576e2 100644 --- a/datafusion/functions/src/core/with_metadata.rs +++ b/datafusion/functions/src/core/with_metadata.rs @@ -157,7 +157,7 @@ impl ScalarUDFImpl for WithMetadataFunc { #[cfg(test)] mod tests { use super::*; - use arrow::datatypes::Field; + use arrow::datatypes::{Field, Metadata}; use datafusion_common::ScalarValue; use std::sync::Arc; @@ -197,14 +197,7 @@ mod tests { fn merges_existing_metadata_and_overwrites_on_collision() { let udf = WithMetadataFunc::new(); let mut existing = Field::new("x", DataType::Float64, false); - existing.set_metadata( - [ - ("keep".to_string(), "yes".to_string()), - ("unit".to_string(), "old".to_string()), - ] - .into_iter() - .collect(), - ); + existing.set_metadata(Metadata::new().with("keep", "yes").with("unit", "old")); let input: FieldRef = Arc::new(existing); let k = str_lit("unit"); let v = str_lit("new"); diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index db7ad8475273a..7a459cd7db4e0 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -783,6 +783,8 @@ fn assert_valid_optimization( mod tests { use std::sync::{Arc, Mutex}; + use arrow::datatypes::Metadata; + use datafusion_common::tree_node::Transformed; use datafusion_common::{ Column, DFSchema, DFSchemaRef, DataFusionError, Result, assert_contains, plan_err, @@ -962,8 +964,7 @@ mod tests { .iter() .enumerate() .map(|(i, (qualifier, field))| { - let metadata = - [("key".into(), format!("value {i}"))].into_iter().collect(); + let metadata = Metadata::new().with("key", format!("value {i}")); let new_arrow_field = field.as_ref().clone().with_metadata(metadata); (qualifier.cloned(), Arc::new(new_arrow_field)) diff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rs index 18ddc361a0692..49e358a854a2f 100644 --- a/datafusion/optimizer/src/propagate_empty_relation.rs +++ b/datafusion/optimizer/src/propagate_empty_relation.rs @@ -348,7 +348,7 @@ fn has_empty_grouping_set(group_expr: &[Expr]) -> bool { #[cfg(test)] mod tests { - use arrow::datatypes::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Metadata, Schema}; use datafusion_common::{Column, DFSchema}; use datafusion_expr::logical_plan::table_scan; @@ -800,7 +800,7 @@ mod tests { produce_one_row: false, schema: Arc::new(DFSchema::from_unqualified_fields( fields.into(), - Default::default(), + Metadata::new(), )?), }); diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index cf54ae254746d..628a8ead99e57 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1437,7 +1437,7 @@ mod tests { use std::cmp::Ordering; use std::fmt::{Debug, Formatter}; - use arrow::datatypes::{Field, Schema, SchemaRef}; + use arrow::datatypes::{Field, Metadata, Schema, SchemaRef}; use async_trait::async_trait; use datafusion_common::{DFSchemaRef, DataFusionError, ScalarValue}; @@ -4263,7 +4263,7 @@ mod tests { let schema = Arc::new( DFSchema::new_with_metadata( vec![(None, Field::new("a", DataType::Int64, false).into())], - Default::default(), + Metadata::new(), ) .unwrap(), ); diff --git a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs index d888a54d56574..c71d17d3034ab 100644 --- a/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs +++ b/datafusion/optimizer/src/simplify_expressions/udf_preimage.rs @@ -71,7 +71,7 @@ mod test { use std::sync::Arc; - use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::{DataType, Field, Metadata}; use datafusion_common::{DFSchema, DFSchemaRef, Result, ScalarValue}; use datafusion_expr::{ ColumnarValue, Expr, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, @@ -201,7 +201,7 @@ mod test { Arc::new( DFSchema::from_unqualified_fields( vec![Field::new("x", DataType::Int32, true)].into(), - Default::default(), + Metadata::new(), ) .unwrap(), ) @@ -215,7 +215,7 @@ mod test { Field::new("y", DataType::Int32, false), ] .into(), - Default::default(), + Metadata::new(), ) .unwrap(), ) diff --git a/datafusion/physical-expr/benches/case_when.rs b/datafusion/physical-expr/benches/case_when.rs index 33931a2ba98e4..88239149e042c 100644 --- a/datafusion/physical-expr/benches/case_when.rs +++ b/datafusion/physical-expr/benches/case_when.rs @@ -18,7 +18,6 @@ use arrow::array::{Array, ArrayRef, Int32Array, Int32Builder, StringArray}; use arrow::datatypes::{ArrowNativeTypeOp, Field, Schema}; use arrow::record_batch::RecordBatch; -use arrow::util::test_util::seedable_rng; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use datafusion_common::ScalarValue; use datafusion_expr::Operator; @@ -28,7 +27,13 @@ use itertools::Itertools; use rand::distr::Alphanumeric; use rand::distr::uniform::SampleUniform; use rand::rngs::StdRng; -use rand::{Rng, RngCore}; +use rand::{Rng, RngCore, SeedableRng}; + +/// Returns a fixed-seed RNG using this crate's `rand` version (arrow's +/// `test_util::seedable_rng` returns its own `rand` version's `StdRng`) +fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} use std::fmt::{Display, Formatter}; use std::hint::black_box; use std::ops::Range; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index f80d1b15bdc59..e784805e2f55e 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -743,7 +743,7 @@ pub fn logical2physical(expr: &Expr, schema: &Schema) -> Arc { #[cfg(test)] mod tests { use arrow::array::{ArrayRef, BooleanArray, RecordBatch, StringArray}; - use arrow::datatypes::{DataType, Field}; + use arrow::datatypes::{DataType, Field, Metadata}; use datafusion_expr::col; use super::*; @@ -803,7 +803,7 @@ mod tests { let schema = test_cast_schema(); let target_field = Arc::new( Field::new("cast_target", DataType::Int64, true) - .with_metadata([("target_meta".to_string(), "1".to_string())].into()), + .with_metadata(Metadata::new().with("target_meta", "1")), ); let cast_expr = Expr::Cast(Cast::new_from_field( Box::new(col("a")), @@ -841,9 +841,8 @@ mod tests { fn test_cast_lowering_preserves_same_type_field_semantics() -> Result<()> { let schema = test_cast_schema(); let target_field = Arc::new( - Field::new("same_type_cast", DataType::Int32, true).with_metadata( - [("target_meta".to_string(), "same-type".to_string())].into(), - ), + Field::new("same_type_cast", DataType::Int32, true) + .with_metadata(Metadata::new().with("target_meta", "same-type")), ); let cast_expr = Expr::Cast(Cast::new_from_field( Box::new(col("a")), diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs index 20467a7ec5e33..3bbd3da322304 100644 --- a/datafusion/physical-plan/src/joins/utils.rs +++ b/datafusion/physical-plan/src/joins/utils.rs @@ -338,12 +338,8 @@ pub fn build_join_schema( _ => (right, left), }; - let metadata = schema1 - .metadata() - .clone() - .into_iter() - .chain(schema2.metadata().clone()) - .collect(); + let mut metadata = schema1.metadata().clone(); + metadata.extend(schema2.metadata().clone()); (fields.finish().with_metadata(metadata), column_indices) } diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rs index aee9e917c755d..a5613515ad59f 100644 --- a/datafusion/physical-plan/src/spill/spill_manager.rs +++ b/datafusion/physical-plan/src/spill/spill_manager.rs @@ -20,8 +20,7 @@ use super::{SpillReaderStream, in_progress_spill_file::InProgressSpillFile}; use crate::coop::cooperative; use crate::{common::spawn_buffered, metrics::SpillMetrics}; -use arrow::array::{BinaryViewArray, GenericByteViewArray, StringViewArray}; -use arrow::datatypes::{ByteViewType, SchemaRef}; +use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use datafusion_common::{DataFusionError, Result, config::SpillCompression}; use datafusion_execution::SendableRecordBatchStream; @@ -220,35 +219,14 @@ impl GetSlicedSize for RecordBatch { let mut total = 0; for array in self.columns() { let data = array.to_data(); + // Since https://github.com/apache/arrow-rs/issues/8230 this also + // accounts for the variadic data buffers retained by view arrays total += data.get_slice_memory_size()?; - - // While StringViewArray holds large data buffer for non inlined string, the Arrow layout (BufferSpec) - // does not include any data buffers. Currently, ArrayData::get_slice_memory_size() - // under-counts memory size by accounting only views buffer although data buffer is cloned during slice() - // - // Therefore, we manually add the sum of the lengths used by all non inlined views - // on top of the sliced size for views buffer. This matches the intended semantics of - // "bytes needed if we materialized exactly this slice into fresh buffers". - // This is a workaround until https://github.com/apache/arrow-rs/issues/8230 - if let Some(sv) = array.as_any().downcast_ref::() { - total += byte_view_data_buffer_size(sv); - } - if let Some(bv) = array.as_any().downcast_ref::() { - total += byte_view_data_buffer_size(bv); - } } Ok(total) } } -fn byte_view_data_buffer_size(array: &GenericByteViewArray) -> usize { - array - .data_buffers() - .iter() - .map(|buffer| buffer.capacity()) - .sum() -} - #[cfg(test)] mod tests { use super::SpillManager; @@ -399,10 +377,11 @@ mod tests { half_batch.get_sliced_size().unwrap() < get_record_batch_memory_size(&half_batch) ); + // Since arrow 60, `get_slice_memory_size` accounts for the retained + // variadic data buffers as well, so it matches `get_sliced_size` let data = arrow::array::Array::to_data(&half_batch.column(0)); let views_sliced_size = data.get_slice_memory_size()?; - // The sliced size should be larger than sliced views buffer size - assert!(views_sliced_size < half_batch.get_sliced_size().unwrap()); + assert_eq!(views_sliced_size, half_batch.get_sliced_size().unwrap()); Ok(()) } diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index c1cc5da31abaf..bf70af1ae89fb 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -46,7 +46,7 @@ use crate::statistics::{ChildStats, StatisticsArgs}; use crate::stream::ObservedStream; use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count}; -use arrow::datatypes::{Field, Schema, SchemaRef}; +use arrow::datatypes::{Field, Metadata, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::config::ConfigOptions; use datafusion_common::stats::NdvFallback; @@ -963,7 +963,7 @@ fn union_schema(inputs: &[Arc]) -> Result { }) .collect::>(); - let all_metadata_merged = inputs + let all_metadata_merged: Metadata = inputs .iter() .flat_map(|i| i.schema().metadata().clone().into_iter()) .collect(); diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index 169ff7f3d9ff2..df583112672aa 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -28,7 +28,7 @@ use arrow::datatypes::{ TimeUnit, UnionFields, UnionMode, i256, }; use arrow::ipc::{ - convert::fb_to_schema, + convert::try_fb_to_schema, reader::{read_dictionary, read_record_batch}, root_as_message, writer::{DictionaryTracker, IpcDataGenerator, IpcWriteOptions}, @@ -461,7 +461,11 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue { .to_string(), ) })?; - fb_to_schema(ipc_schema) + try_fb_to_schema(ipc_schema).map_err(|e| { + Error::General(format!( + "Error converting IPC schema while deserializing nested ScalarValue: {e}" + )) + })? }; let message = root_as_message(ipc_message.as_slice()).map_err(|e| { diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs index 360981746585b..601efbc3d03a5 100644 --- a/datafusion/proto-common/src/to_proto/mod.rs +++ b/datafusion/proto-common/src/to_proto/mod.rs @@ -99,7 +99,7 @@ impl TryFrom<&Field> for protobuf::Field { arrow_type: Some(Box::new(arrow_type)), nullable: field.is_nullable(), children: Vec::new(), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), }) } } @@ -265,7 +265,7 @@ impl TryFrom<&Schema> for protobuf::Schema { fn try_from(schema: &Schema) -> Result { Ok(Self { columns: convert_arc_fields_to_proto_fields(schema.fields())?, - metadata: schema.metadata.clone(), + metadata: schema.metadata().into(), }) } } @@ -276,7 +276,7 @@ impl TryFrom for protobuf::Schema { fn try_from(schema: SchemaRef) -> Result { Ok(Self { columns: convert_arc_fields_to_proto_fields(schema.fields())?, - metadata: schema.metadata.clone(), + metadata: schema.metadata().into(), }) } } @@ -298,7 +298,7 @@ impl TryFrom<&DFSchema> for protobuf::DfSchema { .collect::, Error>>()?; Ok(Self { columns, - metadata: s.metadata().clone(), + metadata: s.metadata().into(), }) } } diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 16c3468465541..8996409f71a2f 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -400,7 +400,7 @@ pub fn serialize_expr( let expr = Box::new(protobuf::CastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), arrow_type: Some(field.data_type().try_into()?), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { @@ -411,7 +411,7 @@ pub fn serialize_expr( let expr = Box::new(protobuf::TryCastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), arrow_type: Some(field.data_type().try_into()?), - metadata: field.metadata().clone(), + metadata: field.metadata().into(), nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { @@ -504,7 +504,7 @@ pub fn serialize_expr( nullable: field.as_ref().map(|f| f.is_nullable()), metadata: field .as_ref() - .map(|f| f.metadata().clone()) + .map(|f| f.metadata().into()) .unwrap_or_default(), })), }, diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index a450f7a7e888f..c63ff45000c2f 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -21,8 +21,8 @@ use arrow::array::{ }; use arrow::datatypes::{ DECIMAL256_MAX_PRECISION, DataType, Field, FieldRef, Fields, Int32Type, - IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Schema, SchemaRef, - TimeUnit, UnionFields, UnionMode, + IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, Metadata, Schema, + SchemaRef, TimeUnit, UnionFields, UnionMode, }; use arrow::util::pretty::pretty_format_batches; use datafusion::datasource::file_format::json::{JsonFormat, JsonFormatFactory}; @@ -1676,9 +1676,7 @@ async fn roundtrip_logical_plan_prepared_statement_with_metadata() -> Result<()> "".to_string(), vec![ Field::new("", DataType::Int32, true) - .with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(), ], ) diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs index 46ab8511f7002..b865940c710e0 100644 --- a/datafusion/proto/tests/cases/serialize.rs +++ b/datafusion/proto/tests/cases/serialize.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{DataType, Field, Metadata}; use datafusion::execution::FunctionRegistry; use datafusion::prelude::SessionContext; @@ -203,9 +203,7 @@ fn roundtrip_placeholder_with_metadata() { "placeholder_id".to_string(), Some( Field::new("", DataType::Utf8, false) - .with_metadata( - [("some_key".to_string(), "some_value".to_string())].into(), - ) + .with_metadata(Metadata::new().with("some_key", "some_value")) .into(), ), )); diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index fd5c34ff5d961..1505070709821 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -29,7 +29,7 @@ use crate::planner::{ }; use crate::utils::normalize_ident; -use arrow::datatypes::{Field, FieldRef, Fields}; +use arrow::datatypes::{Field, FieldRef, Fields, Metadata}; use datafusion_common::error::_plan_err; use datafusion_common::format::ExplainStatementOptions; use datafusion_common::parsers::CompressionTypeVariant; @@ -2928,7 +2928,7 @@ impl SqlToRel<'_, S> { let mut planner_context = PlannerContext::new().with_prepare_param_data_types(prepare_param_data_types); planner_context.set_table_schema(Some(DFSchemaRef::new( - DFSchema::from_unqualified_fields(fields.clone(), Default::default())?, + DFSchema::from_unqualified_fields(fields.clone(), Metadata::new())?, ))); let source = self.query_to_plan(*source, &mut planner_context)?; if fields.len() != source.schema().fields().len() { diff --git a/datafusion/sql/tests/cases/params.rs b/datafusion/sql/tests/cases/params.rs index 68c560ead68cd..41eb6ef8ea4c2 100644 --- a/datafusion/sql/tests/cases/params.rs +++ b/datafusion/sql/tests/cases/params.rs @@ -16,7 +16,7 @@ // under the License. use crate::logical_plan; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Metadata}; use datafusion_common::{ ParamValues, ScalarValue, assert_contains, metadata::{ScalarAndMetadata, format_type_and_metadata}, @@ -732,9 +732,8 @@ fn test_prepare_statement_to_plan_one_param() { fn test_update_infer_with_metadata() { // Here the uuid field is inferred as nullable because it appears in the filter // (and not in the update values, where its nullability would be inferred) - let uuid_field = Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())].into(), - ); + let uuid_field = Field::new("", DataType::FixedSizeBinary(16), true) + .with_metadata(Metadata::new().with("ARROW:extension:name", "arrow.uuid")); let uuid_bytes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let expected_types = vec![ ( @@ -801,9 +800,8 @@ fn test_update_infer_with_metadata() { #[test] fn test_insert_infer_with_metadata() { - let uuid_field = Field::new("", DataType::FixedSizeBinary(16), false).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())].into(), - ); + let uuid_field = Field::new("", DataType::FixedSizeBinary(16), false) + .with_metadata(Metadata::new().with("ARROW:extension:name", "arrow.uuid")); let uuid_bytes = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; let expected_types = vec![ ("$1", Some(uuid_field.clone().with_name("id").into())), diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index e7c819bbf64a6..1a2f82506e82c 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -168,8 +168,7 @@ impl ContextProvider for MockContextProvider { ])), "person_with_uuid_extension" => Ok(Schema::new(vec![ Field::new("id", DataType::FixedSizeBinary(16), false).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), Field::new("first_name", DataType::Utf8, false), Field::new("last_name", DataType::Utf8, false), @@ -382,8 +381,7 @@ impl TypePlanner for CustomTypePlanner { match sql_type { sqlparser::ast::DataType::Uuid => Ok(Some(Arc::new( Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), ))), sqlparser::ast::DataType::Datetime(precision) => { diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index d85ca2db76268..0a551a8f69f47 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -29,7 +29,7 @@ use arrow::array::{ }; use arrow::buffer::ScalarBuffer; use arrow::datatypes::{ - DataType, Field, FieldRef, Fields, Schema, SchemaRef, TimeUnit, UInt32Type, + DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaRef, TimeUnit, UInt32Type, UnionFields, }; use arrow::record_batch::RecordBatch; @@ -82,8 +82,7 @@ impl TypePlanner for SqlLogicTestTypePlanner { match sql_type { ast::DataType::Uuid => Ok(Some(Arc::new( Field::new("", DataType::FixedSizeBinary(16), true).with_metadata( - [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] - .into(), + Metadata::new().with("ARROW:extension:name", "arrow.uuid"), ), ))), _ => Ok(None), diff --git a/datafusion/sqllogictest/test_files/datetime/arith_interval_double.slt b/datafusion/sqllogictest/test_files/datetime/arith_interval_double.slt index d48d2b59c8bee..b431bc4c86721 100644 --- a/datafusion/sqllogictest/test_files/datetime/arith_interval_double.slt +++ b/datafusion/sqllogictest/test_files/datetime/arith_interval_double.slt @@ -18,7 +18,7 @@ query error Invalid interval arithmetic operation: Interval\(MonthDayNano\) \* I SELECT interval '1 day' * 21 -query error Invalid interval arithmetic operation: Interval\(MonthDayNano\) \* Interval\(MonthDayNano\) +query error DataFusion error: Arrow error: Invalid argument error: Invalid interval arithmetic operation: Interval\(MonthDayNano\) \* Float64 SELECT interval '1 hour' * 3.5 @@ -33,9 +33,13 @@ SELECT arrow_typeof(interval '1 second' * 900) # Divide an interval by a scalar # interval '1 hour' / 1.5 → 00:40:00 -query error Invalid interval arithmetic operation: Interval\(MonthDayNano\) / Interval\(MonthDayNano\) +query ? SELECT interval '1 hour' / 1.5 +---- +40 mins -query error Invalid interval arithmetic operation: Interval\(MonthDayNano\) / Interval\(MonthDayNano\) +query T SELECT arrow_typeof(interval '1 hour' / 1.5) +---- +Interval(MonthDayNano) diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 72d034067663e..acfd5164178e5 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -268,7 +268,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1; ---- Plan with Metrics 01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false], filter=[b@1 IS NULL OR b@1 > bd], metrics=[output_rows=1, output_batches=1, row_replacements=1] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true, dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=21.62% (222/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true, dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=21.94% (222/1.01 K)] statement ok reset datafusion.explain.analyze_categories; @@ -319,7 +319,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_multi_col ORDER BY b ASC NULLS LAST, a DESC L ---- Plan with Metrics 01)SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false], filter=[b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac)], metrics=[output_rows=2, output_batches=1, row_replacements=2] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 DESC], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=21.62% (222/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 DESC], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=21.94% (222/1.01 K)] statement ok reset datafusion.explain.analyze_categories; @@ -388,8 +388,8 @@ FROM join_probe p INNER JOIN join_build AS build ---- Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.58% (196/1.00 K)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.05% (228/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] statement ok reset datafusion.explain.analyze_categories; @@ -474,9 +474,9 @@ INNER JOIN nested_t3 ON nested_t2.c = nested_t3.d; Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c@3, d@0)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] 02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, b@0)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t1.parquet]]}, projection=[a, x], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=17.37% (132/760)] -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t2.parquet]]}, projection=[b, c, y], file_type=parquet, predicate=DynamicFilter [ b@0 >= aa AND b@0 <= ab AND b@0 IN (SET) ([aa, ab]) ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= aa AND b_null_count@1 != row_count@2 AND b_min@3 <= ab AND (b_null_count@1 != row_count@2 AND b_min@3 <= aa AND aa <= b_max@0 OR b_null_count@1 != row_count@2 AND b_min@3 <= ab AND ab <= b_max@0), required_guarantees=[b in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=5 total → 5 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=3, predicate_cache_inner_records=5, predicate_cache_records=2, scan_efficiency_ratio=22.46% (234/1.04 K)] -05)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t3.parquet]]}, projection=[d, z], file_type=parquet, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= ca AND d_null_count@1 != row_count@2 AND d_min@3 <= cb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=8 total → 8 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=6, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=21.45% (172/802)] +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t1.parquet]]}, projection=[a, x], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=17.72% (132/745)] +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t2.parquet]]}, projection=[b, c, y], file_type=parquet, predicate=DynamicFilter [ b@0 >= aa AND b@0 <= ab AND b@0 IN (SET) ([aa, ab]) ], dynamic_rg_pruning=eligible, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 >= aa AND b_null_count@1 != row_count@2 AND b_min@3 <= ab AND (b_null_count@1 != row_count@2 AND b_min@3 <= aa AND aa <= b_max@0 OR b_null_count@1 != row_count@2 AND b_min@3 <= ab AND ab <= b_max@0), required_guarantees=[b in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=5 total → 5 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=3, predicate_cache_inner_records=5, predicate_cache_records=2, scan_efficiency_ratio=22.78% (234/1.03 K)] +05)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nested_t3.parquet]]}, projection=[d, z], file_type=parquet, predicate=DynamicFilter [ d@0 >= ca AND d@0 <= cb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= ca AND d_null_count@1 != row_count@2 AND d_min@3 <= cb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=8 total → 8 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=6, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=21.86% (172/787)] statement ok reset datafusion.explain.analyze_categories; @@ -605,8 +605,8 @@ LIMIT 2; Plan with Metrics 01)SortExec: TopK(fetch=2), expr=[e@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[e@0 < bb], metrics=[output_rows=2, output_batches=1, row_replacements=2] 02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, d@0)], projection=[e@2], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_build.parquet]]}, projection=[a], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=6.39% (64/1.00 K)] -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_probe.parquet]]}, projection=[d, e], file_type=parquet, predicate=DynamicFilter [ d@0 >= aa AND d@0 <= ab AND d@0 IN (SET) ([aa, ab]) ] AND DynamicFilter [ e@1 < bb ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= aa AND d_null_count@1 != row_count@2 AND d_min@3 <= ab AND (d_null_count@1 != row_count@2 AND d_min@3 <= aa AND aa <= d_max@0 OR d_null_count@1 != row_count@2 AND d_min@3 <= ab AND ab <= d_max@0) AND e_null_count@5 != row_count@2 AND e_min@4 < bb, required_guarantees=[d in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=14.89% (154/1.03 K)] +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_build.parquet]]}, projection=[a], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=6.49% (64/986)] +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_join_probe.parquet]]}, projection=[d, e], file_type=parquet, predicate=DynamicFilter [ d@0 >= aa AND d@0 <= ab AND d@0 IN (SET) ([aa, ab]) ] AND DynamicFilter [ e@1 < bb ], dynamic_rg_pruning=eligible, pruning_predicate=d_null_count@1 != row_count@2 AND d_max@0 >= aa AND d_null_count@1 != row_count@2 AND d_min@3 <= ab AND (d_null_count@1 != row_count@2 AND d_min@3 <= aa AND aa <= d_max@0 OR d_null_count@1 != row_count@2 AND d_min@3 <= ab AND ab <= d_max@0) AND e_null_count@5 != row_count@2 AND e_min@4 < bb, required_guarantees=[d in (aa, ab)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)] statement ok reset datafusion.explain.analyze_categories; @@ -656,7 +656,7 @@ EXPLAIN ANALYZE SELECT b, a FROM topk_proj ORDER BY a LIMIT 2; Plan with Metrics 01)ProjectionExec: expr=[b@1 as b, a@0 as a], metrics=[output_rows=2, output_batches=1] 02)--SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.21% (141/1.07 K)] +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.4% (141/1.05 K)] # Case 2: prune — `SELECT a` — filter stays as `a < 2` on the scan. query TT @@ -664,7 +664,7 @@ EXPLAIN ANALYZE SELECT a FROM topk_proj ORDER BY a LIMIT 2; ---- Plan with Metrics 01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=6.84% (73/1.07 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=6.94% (73/1.05 K)] # Case 3: expression — `SELECT a+1 AS a_plus_1` — the TopK filter is on # `a_plus_1`, the scan predicate must read `a@0 + 1`. @@ -673,7 +673,7 @@ EXPLAIN ANALYZE SELECT a + 1 AS a_plus_1, b FROM topk_proj ORDER BY a_plus_1 LIM ---- Plan with Metrics 01)SortExec: TopK(fetch=2), expr=[a_plus_1@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a_plus_1@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a_plus_1, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], dynamic_rg_pruning=eligible, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.21% (141/1.07 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a_plus_1, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], dynamic_rg_pruning=eligible, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.4% (141/1.05 K)] # Case 4: alias shadowing — `SELECT a+1 AS a` — the projection renames # `a+1` to `a`, so the TopK's `a < 3` must still be rewritten to @@ -683,7 +683,7 @@ EXPLAIN ANALYZE SELECT a + 1 AS a, b FROM topk_proj ORDER BY a LIMIT 2; ---- Plan with Metrics 01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.21% (141/1.07 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.4% (141/1.05 K)] statement ok reset datafusion.explain.analyze_categories; @@ -745,7 +745,7 @@ Plan with Metrics 04)----AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[min(join_agg_probe.value)], metrics=[output_rows=2, output_batches=2, spill_count=0, spilled_rows=0] 05)------RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1, metrics=[output_rows=2, output_batches=2, spill_count=0, spilled_rows=0] 06)--------AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[min(join_agg_probe.value)], metrics=[output_rows=2, output_batches=1, spill_count=0, spilled_rows=0, skipped_aggregation_rows=0, reduction_factor=100% (2/2)] -07)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_agg_probe.parquet]]}, projection=[a, value], file_type=parquet, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= h1 AND a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND (a_null_count@1 != row_count@2 AND a_min@3 <= h1 AND h1 <= a_max@0 OR a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND h2 <= a_max@0), required_guarantees=[a in (h1, h2)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=2, scan_efficiency_ratio=19.07% (151/792)] +07)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/join_agg_probe.parquet]]}, projection=[a, value], file_type=parquet, predicate=DynamicFilter [ a@0 >= h1 AND a@0 <= h2 AND a@0 IN (SET) ([h1, h2]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= h1 AND a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND (a_null_count@1 != row_count@2 AND a_min@3 <= h1 AND h1 <= a_max@0 OR a_null_count@1 != row_count@2 AND a_min@3 <= h2 AND h2 <= a_max@0), required_guarantees=[a in (h1, h2)], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, page_index_rows_pruned=4 total → 4 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=4, predicate_cache_records=2, scan_efficiency_ratio=19.43% (151/777)] statement ok reset datafusion.explain.analyze_categories; @@ -808,7 +808,7 @@ ON nulls_build.a = nulls_probe.a AND nulls_build.b = nulls_probe.b; Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=1, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=3, input_batches=1, input_rows=1, avg_fanout=100% (1/1), probe_hit_rate=100% (1/1)] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nulls_build.parquet]]}, projection=[a, b], file_type=parquet, metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.6% (144/774)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nulls_probe.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= 1 AND b@1 <= 2 AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:1}, {c0:,c1:2}, {c0:ab,c1:}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= 1 AND b_null_count@5 != row_count@2 AND b_min@6 <= 2, required_guarantees=[], metrics=[output_rows=1, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=1, pushdown_rows_pruned=3, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=20.18% (225/1.11 K)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/nulls_probe.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= 1 AND b@1 <= 2 AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:1}, {c0:,c1:2}, {c0:ab,c1:}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= 1 AND b_null_count@5 != row_count@2 AND b_min@6 <= 2, required_guarantees=[], metrics=[output_rows=1, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=1, pushdown_rows_pruned=3, predicate_cache_inner_records=8, predicate_cache_records=2, scan_efficiency_ratio=20.45% (225/1.10 K)] statement ok reset datafusion.explain.analyze_categories; @@ -873,8 +873,8 @@ ON lj_build.a = lj_probe.a AND lj_build.b = lj_probe.b; ---- Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.58% (196/1.00 K)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.05% (228/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] # LEFT SEMI JOIN: only matching build rows are returned; probe scan still # receives the dynamic filter. @@ -889,8 +889,8 @@ WHERE EXISTS ( ---- Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(a@0, a@0), (b@1, b@1)], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=2, input_rows=4, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.58% (196/1.00 K)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=14.89% (154/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/lj_probe.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND struct(a@0, b@1) IN (SET) ([{c0:aa,c1:ba}, {c0:ab,c1:bb}]) ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=15.11% (154/1.02 K)] statement ok reset datafusion.explain.analyze_categories; @@ -959,8 +959,8 @@ FROM hl_probe p INNER JOIN hl_build AS build ---- Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0), (b@1, b@1)], projection=[a@3, b@4, c@2, e@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.58% (196/1.00 K)] -03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.05% (228/1.03 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_build.parquet]]}, projection=[a, b, c], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=19.88% (196/986)] +03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/hl_probe.parquet]]}, projection=[a, b, e], file_type=parquet, predicate=DynamicFilter [ a@0 >= aa AND a@0 <= ab AND b@1 >= ba AND b@1 <= bb AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 >= aa AND a_null_count@1 != row_count@2 AND a_min@3 <= ab AND b_null_count@5 != row_count@2 AND b_max@4 >= ba AND b_null_count@5 != row_count@2 AND b_min@6 <= bb, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=22.37% (228/1.02 K)] statement ok drop table hl_build; @@ -1008,7 +1008,7 @@ FROM int_build b INNER JOIN int_probe p ---- Plan with Metrics 01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id1@0, id1@0), (id2@1, id2@1)], projection=[id1@0, id2@1, value@2, data@5], metrics=[output_rows=2, output_batches=1, array_map_created_count=0, build_input_batches=1, build_input_rows=2, input_batches=1, input_rows=2, avg_fanout=100% (2/2), probe_hit_rate=100% (2/2)] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_build.parquet]]}, projection=[id1, id2, value], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.23% (204/1.12 K)] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_build.parquet]]}, projection=[id1, id2, value], file_type=parquet, metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0, predicate_cache_inner_records=0, predicate_cache_records=0, scan_efficiency_ratio=18.48% (204/1.10 K)] 03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/int_probe.parquet]]}, projection=[id1, id2, data], file_type=parquet, predicate=DynamicFilter [ id1@0 >= 1 AND id1@0 <= 2 AND id2@1 >= 10 AND id2@1 <= 20 AND hash_lookup ], dynamic_rg_pruning=eligible, pruning_predicate=id1_null_count@1 != row_count@2 AND id1_max@0 >= 1 AND id1_null_count@1 != row_count@2 AND id1_min@3 <= 2 AND id2_null_count@5 != row_count@2 AND id2_max@4 >= 10 AND id2_null_count@5 != row_count@2 AND id2_min@6 <= 20, required_guarantees=[], metrics=[output_rows=2, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=2, pushdown_rows_pruned=2, predicate_cache_inner_records=8, predicate_cache_records=4, scan_efficiency_ratio=20.67% (221/1.07 K)] statement ok