From 53f25d3980ae52686f8d419662de248e5ba3b086 Mon Sep 17 00:00:00 2001 From: Dharan Aditya Date: Tue, 8 Jul 2025 14:32:52 +0530 Subject: [PATCH 1/5] feat: Upgrade to the official DataFusion 49.0.0 release --- native/Cargo.lock | 532 +++++++++++++++++++++++++++++------------ native/Cargo.toml | 4 +- native/core/Cargo.toml | 2 +- 3 files changed, 382 insertions(+), 156 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index b6e4c0e0f7..4aaaf64f8f 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -244,6 +244,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -1264,8 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6cb8c2c81eada072059983657d6c9caf3fddefc43b4a65551d243253254a96" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "arrow-ipc", @@ -1275,23 +1275,23 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-nested 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1314,18 +1314,17 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7be8d1b627843af62e447396db08fe1372d882c0eb8d0ea655fd1fbc33120ee" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-session", "datafusion-sql", @@ -1340,18 +1339,17 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ab16c5ae43f65ee525fc493ceffbc41f40dee38b01f643dfcfc12959e92038" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1376,7 +1374,7 @@ dependencies = [ "datafusion-comet-objectstore-hdfs", "datafusion-comet-proto", "datafusion-comet-spark-expr", - "datafusion-functions-nested", + "datafusion-functions-nested 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-spark", "futures", "hex", @@ -1464,6 +1462,28 @@ dependencies = [ "indexmap", "libc", "log", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "base64", + "chrono", + "half", + "hashbrown 0.14.5", + "hex", + "indexmap", + "libc", + "log", "object_store", "parquet", "paste", @@ -1475,8 +1495,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16015071202d6133bc84d72756176467e3e46029f3ce9ad2cb788f9b1ff139b2" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "futures", "log", @@ -1486,19 +1505,18 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b77523c95c89d2a7eb99df14ed31390e04ab29b43ff793e562bdc1716b07e17b" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1516,20 +1534,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d25c5e2c0ebe8434beeea997b8e88d55b3ccc0d19344293f2373f65bc524fc" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1541,20 +1558,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc6959e1155741ab35369e1dc7673ba30fc45ed568fad34c01b7cb1daeb4d4c" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1566,23 +1582,23 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a6afdfe358d70f4237f60eaef26ae5a1ce7cb2c469d02d5fc6c7fd5d84e58b" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", "itertools 0.14.0", @@ -1600,6 +1616,11 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bcd8a3e3e3d02ea642541be23d44376b5d5c37c2938cce39b3873cdf7186eea" +[[package]] +name = "datafusion-doc" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" + [[package]] name = "datafusion-execution" version = "48.0.0" @@ -1608,8 +1629,26 @@ checksum = "670da1d45d045eee4c2319b8c7ea57b26cf48ab77b630aaa50b779e406da476a" dependencies = [ "arrow", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.1", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "arrow", + "dashmap", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "futures", "log", "object_store", @@ -1627,12 +1666,32 @@ checksum = "b3a577f64bdb7e2cc4043cd97f8901d8c504711fde2dbcb0887645b00d7c660b" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "indexmap", "paste", "serde_json", @@ -1646,7 +1705,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b7916806ace3e9f41884f230f7f38ebf0e955dfbd88266da1826f29a0b9a6a" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "indexmap", "itertools 0.14.0", "paste", @@ -1657,6 +1728,30 @@ name = "datafusion-functions" version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb31c9dc73d3e0c365063f91139dc273308f8a8e124adda9898db8085d68357" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex", + "itertools 0.14.0", + "log", + "rand 0.9.1", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "arrow-buffer", @@ -1664,12 +1759,12 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "hex", "itertools 0.14.0", "log", @@ -1689,14 +1784,34 @@ checksum = "ebb72c6940697eaaba9bd1f746a697a07819de952b817e3fb841fb75331ad5d4" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "half", "log", "paste", @@ -1710,9 +1825,21 @@ checksum = "d7fdc54656659e5ecd49bf341061f4156ab230052611f4f3609612a0da259696" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", ] [[package]] @@ -1723,14 +1850,35 @@ checksum = "fad94598e3374938ca43bca6b675febe557e7a14eb627d617db427d70d65118b" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.14.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-nested" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "itertools 0.14.0", "log", "paste", @@ -1739,14 +1887,13 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2fc6c2946da5cab8364fb28b5cac3115f0f3a87960b235ed031c3f7e2e639b" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "parking_lot", "paste", @@ -1755,17 +1902,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5746548a8544870a119f556543adcd88fe0ba6b93723fe78ad0439e0fbb8b4" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "log", "paste", ] @@ -1776,8 +1922,17 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbe9404382cda257c434f22e13577bee7047031dfdb6216dd5e841b9465e6fe" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", ] [[package]] @@ -1786,7 +1941,17 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8dce50e3b637dab0d25d04d2fe79dfdca2b257eabd76790bffd22c7f90d700c8" dependencies = [ - "datafusion-expr", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "datafusion-macros" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "quote", "syn 2.0.104", ] @@ -1794,14 +1959,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfaacf06445dc3bbc1e901242d2a44f2cae99a744f49f3fefddcee46240058" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "indexmap", "itertools 0.14.0", "log", @@ -1817,11 +1982,32 @@ checksum = "1908034a89d7b2630898e06863583ae4c00a0dd310c1589ca284195ee3f7f8a6" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "log", + "paste", + "petgraph 0.8.2", +] + +[[package]] +name = "datafusion-physical-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "half", "hashbrown 0.14.5", "indexmap", @@ -1839,8 +2025,21 @@ checksum = "47b7a12dd59ea07614b67dbb01d85254fbd93df45bcffa63495e11d3bdf847df" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -1848,17 +2047,17 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4371cc4ad33978cc2a8be93bd54a232d3f2857b50401a14631c0705f3f910aae" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", ] @@ -1866,8 +2065,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc47bc33025757a5c11f2cd094c5b6b5ed87f46fa33c023e6fdfa25fcbfade23" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "ahash", "arrow", @@ -1875,13 +2073,13 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "futures", "half", "hashbrown 0.14.5", @@ -1893,20 +2091,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-pruning" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-datasource", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7485da32283985d6b45bd7d13a65169dcbe8c869e25d01b2cfbc425254b4b49" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "datafusion-physical-plan", "datafusion-sql", "futures", @@ -1920,29 +2134,27 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00825d8e84c5a96e21ea5e525a6a4ec8eea7d623942f13031b519ca596802e90" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-macros", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "log", ] [[package]] name = "datafusion-sql" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a466b15632befddfeac68c125f0260f569ff315c6831538cbb40db754134e0df" +source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" dependencies = [ "arrow", "bigdecimal", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", "indexmap", "log", "regex", @@ -2673,6 +2885,17 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -3306,6 +3529,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -4522,17 +4746,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", diff --git a/native/Cargo.toml b/native/Cargo.toml index 54e459646c..7074ba7b45 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,8 +38,8 @@ arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.10.0" } parquet = { version = "55.1.0", default-features = false, features = ["experimental"] } -datafusion = { version = "48.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-spark = { version = "48.0.0" } +datafusion = { git = "https://github.com/apache/datafusion", rev = "a089eff", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "a089eff" } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index e248068b35..dce5afa3ec 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -41,7 +41,7 @@ futures = { workspace = true } mimalloc = { version = "*", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls"] } tikv-jemalloc-ctl = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls", "stats"] } -tokio = { version = "1", features = ["rt-multi-thread"] } +tokio = { version = "1.46.1", features = ["rt-multi-thread"] } async-trait = { workspace = true } log = "0.4" log4rs = "1.2.0" From 6966b6d72208e3b3a78157a66c5f96a0f4d1a330 Mon Sep 17 00:00:00 2001 From: Dharan Aditya Date: Tue, 8 Jul 2025 15:50:11 +0530 Subject: [PATCH 2/5] fix compilation issues --- native/core/src/execution/operators/filter.rs | 21 +++++++++---------- native/core/src/execution/planner.rs | 21 ++++++++----------- .../src/execution/shuffle/shuffle_writer.rs | 6 +++--- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index 573287a088..deafdafc26 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -211,21 +211,21 @@ impl FilterExec { if let Some(binary) = conjunction.as_any().downcast_ref::() { if binary.op() == &Operator::Eq { // Filter evaluates to single value for all partitions - if input_eqs.is_expr_constant(binary.left()) { + if input_eqs.is_expr_constant(binary.left()).is_some() { let (expr, across_parts) = ( binary.right(), - input_eqs.get_expr_constant_value(binary.right()), + input_eqs.is_expr_constant(binary.right()).unwrap(), ); res_constants.push( - ConstExpr::new(Arc::clone(expr)).with_across_partitions(across_parts), + ConstExpr::new(Arc::clone(expr), across_parts), ); - } else if input_eqs.is_expr_constant(binary.right()) { + } else if input_eqs.is_expr_constant(binary.right()).is_some() { let (expr, across_parts) = ( binary.left(), - input_eqs.get_expr_constant_value(binary.left()), + input_eqs.is_expr_constant(binary.left()).unwrap(), ); res_constants.push( - ConstExpr::new(Arc::clone(expr)).with_across_partitions(across_parts), + ConstExpr::new(Arc::clone(expr), across_parts), ); } } @@ -246,7 +246,7 @@ impl FilterExec { let mut eq_properties = input.equivalence_properties().clone(); let (equal_pairs, _) = collect_columns_from_predicate(predicate); for (lhs, rhs) in equal_pairs { - eq_properties.add_equal_conditions(lhs, rhs)? + eq_properties.add_equal_conditions(lhs.clone(), rhs.clone())? } // Add the columns that have only one viable value (singleton) after // filtering to constants. @@ -258,14 +258,13 @@ impl FilterExec { .min_value .get_value(); let expr = Arc::new(column) as _; - ConstExpr::new(expr) - .with_across_partitions(AcrossPartitions::Uniform(value.cloned())) + ConstExpr::new(expr, AcrossPartitions::Uniform(value.cloned())) }); // This is for statistics - eq_properties = eq_properties.with_constants(constants); + eq_properties.add_constants(constants)?; // This is for logical constant (for example: a = '1', then a could be marked as a constant) // to do: how to deal with multiple situation to represent = (for example c1 between 0 and 0) - eq_properties = eq_properties.with_constants(Self::extend_constants(input, predicate)); + eq_properties.add_constants(Self::extend_constants(input, predicate))?; let mut output_partitioning = input.output_partitioning().clone(); // If contains projection, update the PlanProperties. diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index a45036be45..57aeace9bd 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -73,10 +73,7 @@ use crate::execution::shuffle::{CometPartitioning, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; use crate::parquet::parquet_support::prepare_object_store_with_configs; use datafusion::common::scalar::ScalarStructBuilder; -use datafusion::common::{ - tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, - JoinType as DFJoinType, ScalarValue, -}; +use datafusion::common::{tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, JoinType as DFJoinType, NullEquality, ScalarValue}; use datafusion::datasource::listing::PartitionedFile; use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression; use datafusion::logical_expr::{ @@ -1115,7 +1112,7 @@ impl PhysicalPlanner { let child_copied = Self::wrap_in_copy_exec(Arc::clone(&child.native_plan)); let sort = Arc::new( - SortExec::new(LexOrdering::new(exprs?), Arc::clone(&child_copied)) + SortExec::new(LexOrdering::new(exprs?).expect(""), Arc::clone(&child_copied)) .with_fetch(fetch), ); @@ -1391,7 +1388,7 @@ impl PhysicalPlanner { sort_options, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - false, + NullEquality::NullEqualsNull, )?); if join.filter.is_some() { @@ -1459,7 +1456,7 @@ impl PhysicalPlanner { PartitionMode::Partitioned, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - false, + NullEquality::NullEqualsNull, )?); // If the hash join is build right, we need to swap the left and right @@ -2163,7 +2160,7 @@ impl PhysicalPlanner { window_func_name, &window_args, partition_by, - &LexOrdering::new(sort_exprs.to_vec()), + &LexOrdering::new(sort_exprs.to_vec()).expect("Cannot create lex ordering"), window_frame.into(), input_schema.as_ref(), false, // TODO: Ignore nulls @@ -2244,7 +2241,7 @@ impl PhysicalPlanner { .iter() .map(|expr| self.create_sort_expr(expr, Arc::clone(&input_schema))) .collect(); - let lex_ordering = LexOrdering::from(exprs?); + let lex_ordering = LexOrdering::new(exprs?).expect("Failed to creat"); Ok(CometPartitioning::RangePartitioning( lex_ordering, range_partition.num_partitions as usize, @@ -2614,6 +2611,8 @@ mod tests { use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; use datafusion::{assert_batches_eq, physical_plan::common::collect, prelude::SessionContext}; + use datafusion::functions_array::make_array::make_array_udf; + use datafusion::functions_nested::make_array::MakeArray; use tempfile::TempDir; use tokio::sync::mpsc; @@ -2940,9 +2939,7 @@ mod tests { #[test] fn test_create_array() { let session_ctx = SessionContext::new(); - session_ctx.register_udf(ScalarUDF::from( - datafusion_functions_nested::make_array::MakeArray::new(), - )); + session_ctx.register_udf(ScalarUDF::from(MakeArray::new())); let task_ctx = session_ctx.task_ctx(); let planner = PhysicalPlanner::new(Arc::from(session_ctx), 0); diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index d23d02a2c8..c2233c60c3 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -934,7 +934,7 @@ impl SinglePartitionShufflePartitioner { Ok(Some(concatenated)) } Err(e) => Err(DataFusionError::ArrowError( - e, + Box::new(e), Some(DataFusionError::get_back_trace()), )), } @@ -1122,7 +1122,7 @@ impl Iterator for PartitionedBatchIterator<'_> { Some(Ok(batch)) } Err(e) => Some(Err(DataFusionError::ArrowError( - e, + Box::new(e), Some(DataFusionError::get_back_trace()), ))), } @@ -1409,7 +1409,7 @@ mod test { CometPartitioning::RangePartitioning( LexOrdering::new(vec![PhysicalSortExpr::new_default( col("a", batch.schema().as_ref()).unwrap(), - )]), + )]).expect("Failed to create lex ordering"), num_partitions, 100, ), From 09271f7c981e01049f67442590c29208fab59179 Mon Sep 17 00:00:00 2001 From: Dharan Aditya Date: Tue, 15 Jul 2025 16:10:05 +0530 Subject: [PATCH 3/5] chore: Update Cargo.lock to use DataFusion revision 3291e4e --- native/Cargo.lock | 308 +++++++++--------- native/Cargo.toml | 4 +- native/core/benches/shuffle_writer.rs | 3 +- native/core/src/execution/operators/filter.rs | 8 +- native/core/src/execution/planner.rs | 18 +- .../execution/shuffle/range_partitioner.rs | 3 +- .../src/execution/shuffle/shuffle_writer.rs | 3 +- 7 files changed, 174 insertions(+), 173 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 4aaaf64f8f..b9b929f2f0 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1265,7 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-ipc", @@ -1275,23 +1275,23 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-nested 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-nested 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1314,17 +1314,17 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "datafusion-sql", @@ -1339,17 +1339,17 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-datasource", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1471,7 +1471,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", @@ -1495,7 +1495,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "futures", "log", @@ -1505,18 +1505,18 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1534,19 +1534,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1558,19 +1558,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1582,20 +1582,20 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-pruning", @@ -1619,7 +1619,7 @@ checksum = "9bcd8a3e3e3d02ea642541be23d44376b5d5c37c2938cce39b3873cdf7186eea" [[package]] name = "datafusion-doc" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" [[package]] name = "datafusion-execution" @@ -1643,12 +1643,12 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "futures", "log", "object_store", @@ -1681,17 +1681,17 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "paste", "serde_json", @@ -1714,10 +1714,10 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "itertools 0.14.0", "paste", @@ -1751,7 +1751,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-buffer", @@ -1759,12 +1759,12 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "hex", "itertools 0.14.0", "log", @@ -1800,18 +1800,18 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "half", "log", "paste", @@ -1833,13 +1833,13 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", ] [[package]] @@ -1866,19 +1866,19 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-ord", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "itertools 0.14.0", "log", "paste", @@ -1887,13 +1887,13 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "parking_lot", "paste", @@ -1902,16 +1902,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "log", "paste", ] @@ -1929,10 +1929,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", ] [[package]] @@ -1949,9 +1949,9 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "quote", "syn 2.0.104", ] @@ -1959,14 +1959,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "itertools 0.14.0", "log", @@ -1999,15 +1999,15 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "half", "hashbrown 0.14.5", "indexmap", @@ -2034,12 +2034,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -2047,15 +2047,15 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", @@ -2065,7 +2065,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", @@ -2073,13 +2073,13 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "futures", "half", "hashbrown 0.14.5", @@ -2094,15 +2094,15 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-schema", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-datasource", - "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "itertools 0.14.0", "log", @@ -2111,16 +2111,16 @@ dependencies = [ [[package]] name = "datafusion-session" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-sql", "futures", @@ -2134,27 +2134,27 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "datafusion-catalog", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "log", ] [[package]] name = "datafusion-sql" version = "48.0.0" -source = "git+https://github.com/apache/datafusion?rev=a089eff#a089eff251f13ac52ba070dbdf67cc20d7851662" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "bigdecimal", - "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", - "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=a089eff)", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "log", "regex", diff --git a/native/Cargo.toml b/native/Cargo.toml index 7074ba7b45..2596e575f7 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,8 +38,8 @@ arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.10.0" } parquet = { version = "55.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/apache/datafusion", rev = "a089eff", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "a089eff" } +datafusion = { git = "https://github.com/apache/datafusion", rev = "3291e4e", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "3291e4e" } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/benches/shuffle_writer.rs b/native/core/benches/shuffle_writer.rs index ee07462d00..52638d92a9 100644 --- a/native/core/benches/shuffle_writer.rs +++ b/native/core/benches/shuffle_writer.rs @@ -89,7 +89,8 @@ fn criterion_benchmark(c: &mut Criterion) { CometPartitioning::RangePartitioning( LexOrdering::new(vec![PhysicalSortExpr::new_default( col("c0", batch.schema().as_ref()).unwrap(), - )]), + )]) + .unwrap(), 16, 100, ), diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index deafdafc26..a6d98954dc 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -216,17 +216,13 @@ impl FilterExec { binary.right(), input_eqs.is_expr_constant(binary.right()).unwrap(), ); - res_constants.push( - ConstExpr::new(Arc::clone(expr), across_parts), - ); + res_constants.push(ConstExpr::new(Arc::clone(expr), across_parts)); } else if input_eqs.is_expr_constant(binary.right()).is_some() { let (expr, across_parts) = ( binary.left(), input_eqs.is_expr_constant(binary.left()).unwrap(), ); - res_constants.push( - ConstExpr::new(Arc::clone(expr), across_parts), - ); + res_constants.push(ConstExpr::new(Arc::clone(expr), across_parts)); } } } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 57aeace9bd..afb9c61d19 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -73,7 +73,10 @@ use crate::execution::shuffle::{CometPartitioning, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; use crate::parquet::parquet_support::prepare_object_store_with_configs; use datafusion::common::scalar::ScalarStructBuilder; -use datafusion::common::{tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, JoinType as DFJoinType, NullEquality, ScalarValue}; +use datafusion::common::{ + tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, + JoinType as DFJoinType, NullEquality, ScalarValue, +}; use datafusion::datasource::listing::PartitionedFile; use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression; use datafusion::logical_expr::{ @@ -1112,7 +1115,7 @@ impl PhysicalPlanner { let child_copied = Self::wrap_in_copy_exec(Arc::clone(&child.native_plan)); let sort = Arc::new( - SortExec::new(LexOrdering::new(exprs?).expect(""), Arc::clone(&child_copied)) + SortExec::new(LexOrdering::new(exprs?).unwrap(), Arc::clone(&child_copied)) .with_fetch(fetch), ); @@ -1388,7 +1391,7 @@ impl PhysicalPlanner { sort_options, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - NullEquality::NullEqualsNull, + NullEquality::NullEqualsNothing, )?); if join.filter.is_some() { @@ -1456,7 +1459,7 @@ impl PhysicalPlanner { PartitionMode::Partitioned, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - NullEquality::NullEqualsNull, + NullEquality::NullEqualsNothing, )?); // If the hash join is build right, we need to swap the left and right @@ -2160,7 +2163,7 @@ impl PhysicalPlanner { window_func_name, &window_args, partition_by, - &LexOrdering::new(sort_exprs.to_vec()).expect("Cannot create lex ordering"), + &LexOrdering::new(sort_exprs.to_vec()).unwrap(), window_frame.into(), input_schema.as_ref(), false, // TODO: Ignore nulls @@ -2241,7 +2244,7 @@ impl PhysicalPlanner { .iter() .map(|expr| self.create_sort_expr(expr, Arc::clone(&input_schema))) .collect(); - let lex_ordering = LexOrdering::new(exprs?).expect("Failed to creat"); + let lex_ordering = LexOrdering::new(exprs?).unwrap(); Ok(CometPartitioning::RangePartitioning( lex_ordering, range_partition.num_partitions as usize, @@ -2608,11 +2611,10 @@ mod tests { FileGroup, FileScanConfigBuilder, FileSource, ParquetSource, }; use datafusion::error::DataFusionError; + use datafusion::functions_nested::make_array::MakeArray; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; use datafusion::{assert_batches_eq, physical_plan::common::collect, prelude::SessionContext}; - use datafusion::functions_array::make_array::make_array_udf; - use datafusion::functions_nested::make_array::MakeArray; use tempfile::TempDir; use tokio::sync::mpsc; diff --git a/native/core/src/execution/shuffle/range_partitioner.rs b/native/core/src/execution/shuffle/range_partitioner.rs index 8f443232f9..75dcc4bff9 100644 --- a/native/core/src/execution/shuffle/range_partitioner.rs +++ b/native/core/src/execution/shuffle/range_partitioner.rs @@ -243,7 +243,8 @@ mod test { let lex_ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default( col("a", input_batch.schema().as_ref()).unwrap(), - )]); + )]) + .unwrap(); let (rows, row_converter) = RangePartitioner::generate_bounds( input_batch.columns().to_vec().as_ref(), diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index c2233c60c3..4b712c48bd 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -1409,7 +1409,8 @@ mod test { CometPartitioning::RangePartitioning( LexOrdering::new(vec![PhysicalSortExpr::new_default( col("a", batch.schema().as_ref()).unwrap(), - )]).expect("Failed to create lex ordering"), + )]) + .unwrap(), num_partitions, 100, ), From d36a957d327673f53ea8fd04f674f16de9ccb513 Mon Sep 17 00:00:00 2001 From: Dharan Aditya Date: Wed, 16 Jul 2025 13:51:01 +0530 Subject: [PATCH 4/5] more fixes --- native/core/src/execution/operators/filter.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index a6d98954dc..12d8ccdcd1 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -212,17 +212,17 @@ impl FilterExec { if binary.op() == &Operator::Eq { // Filter evaluates to single value for all partitions if input_eqs.is_expr_constant(binary.left()).is_some() { - let (expr, across_parts) = ( - binary.right(), - input_eqs.is_expr_constant(binary.right()).unwrap(), - ); - res_constants.push(ConstExpr::new(Arc::clone(expr), across_parts)); + let across = input_eqs + .is_expr_constant(binary.right()) + .unwrap_or_default(); + res_constants + .push(ConstExpr::new(Arc::clone(binary.right()), across)); } else if input_eqs.is_expr_constant(binary.right()).is_some() { - let (expr, across_parts) = ( - binary.left(), - input_eqs.is_expr_constant(binary.left()).unwrap(), - ); - res_constants.push(ConstExpr::new(Arc::clone(expr), across_parts)); + let across = input_eqs + .is_expr_constant(binary.left()) + .unwrap_or_default(); + res_constants + .push(ConstExpr::new(Arc::clone(binary.left()), across)); } } } @@ -242,7 +242,7 @@ impl FilterExec { let mut eq_properties = input.equivalence_properties().clone(); let (equal_pairs, _) = collect_columns_from_predicate(predicate); for (lhs, rhs) in equal_pairs { - eq_properties.add_equal_conditions(lhs.clone(), rhs.clone())? + eq_properties.add_equal_conditions(Arc::clone(lhs), Arc::clone(rhs))?; } // Add the columns that have only one viable value (singleton) after // filtering to constants. From 76328709476dfd22dc33524327e03643ee518dc5 Mon Sep 17 00:00:00 2001 From: Dharan Aditya Date: Wed, 16 Jul 2025 19:13:37 +0530 Subject: [PATCH 5/5] formatting --- native/core/src/execution/operators/filter.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index 12d8ccdcd1..c8fcda19fe 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -215,14 +215,12 @@ impl FilterExec { let across = input_eqs .is_expr_constant(binary.right()) .unwrap_or_default(); - res_constants - .push(ConstExpr::new(Arc::clone(binary.right()), across)); + res_constants.push(ConstExpr::new(Arc::clone(binary.right()), across)); } else if input_eqs.is_expr_constant(binary.right()).is_some() { let across = input_eqs .is_expr_constant(binary.left()) .unwrap_or_default(); - res_constants - .push(ConstExpr::new(Arc::clone(binary.left()), across)); + res_constants.push(ConstExpr::new(Arc::clone(binary.left()), across)); } } }