diff --git a/native/Cargo.lock b/native/Cargo.lock index b6e4c0e0f7..b9b929f2f0 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -244,6 +244,7 @@ dependencies = [ "arrow-schema", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -1264,8 +1265,7 @@ dependencies = [ [[package]] name = "datafusion" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6cb8c2c81eada072059983657d6c9caf3fddefc43b4a65551d243253254a96" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-ipc", @@ -1275,23 +1275,23 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-nested 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1314,18 +1314,17 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7be8d1b627843af62e447396db08fe1372d882c0eb8d0ea655fd1fbc33120ee" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "datafusion-sql", @@ -1340,18 +1339,17 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ab16c5ae43f65ee525fc493ceffbc41f40dee38b01f643dfcfc12959e92038" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1376,7 +1374,7 @@ dependencies = [ "datafusion-comet-objectstore-hdfs", "datafusion-comet-proto", "datafusion-comet-spark-expr", - "datafusion-functions-nested", + "datafusion-functions-nested 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "datafusion-spark", "futures", "hex", @@ -1464,6 +1462,28 @@ dependencies = [ "indexmap", "libc", "log", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "base64", + "chrono", + "half", + "hashbrown 0.14.5", + "hex", + "indexmap", + "libc", + "log", "object_store", "parquet", "paste", @@ -1475,8 +1495,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16015071202d6133bc84d72756176467e3e46029f3ce9ad2cb788f9b1ff139b2" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "futures", "log", @@ -1486,19 +1505,18 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b77523c95c89d2a7eb99df14ed31390e04ab29b43ff793e562bdc1716b07e17b" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1516,20 +1534,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d25c5e2c0ebe8434beeea997b8e88d55b3ccc0d19344293f2373f65bc524fc" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1541,20 +1558,19 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc6959e1155741ab35369e1dc7673ba30fc45ed568fad34c01b7cb1daeb4d4c" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1566,23 +1582,23 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a6afdfe358d70f4237f60eaef26ae5a1ce7cb2c469d02d5fc6c7fd5d84e58b" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "bytes", "datafusion-catalog", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-pruning", "datafusion-session", "futures", "itertools 0.14.0", @@ -1600,6 +1616,11 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bcd8a3e3e3d02ea642541be23d44376b5d5c37c2938cce39b3873cdf7186eea" +[[package]] +name = "datafusion-doc" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" + [[package]] name = "datafusion-execution" version = "48.0.0" @@ -1608,8 +1629,26 @@ checksum = "670da1d45d045eee4c2319b8c7ea57b26cf48ab77b630aaa50b779e406da476a" dependencies = [ "arrow", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.1", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "arrow", + "dashmap", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "futures", "log", "object_store", @@ -1627,12 +1666,32 @@ checksum = "b3a577f64bdb7e2cc4043cd97f8901d8c504711fde2dbcb0887645b00d7c660b" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-window-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "paste", "serde_json", @@ -1646,7 +1705,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b7916806ace3e9f41884f230f7f38ebf0e955dfbd88266da1826f29a0b9a6a" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indexmap", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "itertools 0.14.0", "paste", @@ -1657,6 +1728,30 @@ name = "datafusion-functions" version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb31c9dc73d3e0c365063f91139dc273308f8a8e124adda9898db8085d68357" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex", + "itertools 0.14.0", + "log", + "rand 0.9.1", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "arrow-buffer", @@ -1664,12 +1759,12 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "hex", "itertools 0.14.0", "log", @@ -1689,14 +1784,34 @@ checksum = "ebb72c6940697eaaba9bd1f746a697a07819de952b817e3fb841fb75331ad5d4" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "half", "log", "paste", @@ -1710,9 +1825,21 @@ checksum = "d7fdc54656659e5ecd49bf341061f4156ab230052611f4f3609612a0da259696" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", ] [[package]] @@ -1723,14 +1850,35 @@ checksum = "fad94598e3374938ca43bca6b675febe557e7a14eb627d617db427d70d65118b" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-doc 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-execution 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-macros 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.14.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-nested" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "itertools 0.14.0", "log", "paste", @@ -1739,14 +1887,13 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de2fc6c2946da5cab8364fb28b5cac3115f0f3a87960b235ed031c3f7e2e639b" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "parking_lot", "paste", @@ -1755,17 +1902,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5746548a8544870a119f556543adcd88fe0ba6b93723fe78ad0439e0fbb8b4" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-doc 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "log", "paste", ] @@ -1776,8 +1922,17 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbe9404382cda257c434f22e13577bee7047031dfdb6216dd5e841b9465e6fe" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", ] [[package]] @@ -1786,7 +1941,17 @@ version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8dce50e3b637dab0d25d04d2fe79dfdca2b257eabd76790bffd22c7f90d700c8" dependencies = [ - "datafusion-expr", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "datafusion-macros" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "quote", "syn 2.0.104", ] @@ -1794,14 +1959,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfaacf06445dc3bbc1e901242d2a44f2cae99a744f49f3fefddcee46240058" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "itertools 0.14.0", "log", @@ -1817,11 +1982,32 @@ checksum = "1908034a89d7b2630898e06863583ae4c00a0dd310c1589ca284195ee3f7f8a6" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-functions-aggregate-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-physical-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools 0.14.0", + "log", + "paste", + "petgraph 0.8.2", +] + +[[package]] +name = "datafusion-physical-expr" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-aggregate-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "half", "hashbrown 0.14.5", "indexmap", @@ -1839,8 +2025,21 @@ checksum = "47b7a12dd59ea07614b67dbb01d85254fbd93df45bcffa63495e11d3bdf847df" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "datafusion-expr-common 48.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "hashbrown 0.14.5", "itertools 0.14.0", ] @@ -1848,17 +2047,17 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4371cc4ad33978cc2a8be93bd54a232d3f2857b50401a14631c0705f3f910aae" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", "log", ] @@ -1866,8 +2065,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc47bc33025757a5c11f2cd094c5b6b5ed87f46fa33c023e6fdfa25fcbfade23" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "ahash", "arrow", @@ -1875,13 +2073,13 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions-window-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "futures", "half", "hashbrown 0.14.5", @@ -1893,20 +2091,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-pruning" +version = "48.0.0" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-datasource", + "datafusion-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + [[package]] name = "datafusion-session" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7485da32283985d6b45bd7d13a65169dcbe8c869e25d01b2cfbc425254b4b49" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-physical-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "datafusion-physical-plan", "datafusion-sql", "futures", @@ -1920,29 +2134,27 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00825d8e84c5a96e21ea5e525a6a4ec8eea7d623942f13031b519ca596802e90" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-macros", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-execution 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-functions 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-macros 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "log", ] [[package]] name = "datafusion-sql" version = "48.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a466b15632befddfeac68c125f0260f569ff315c6831538cbb40db754134e0df" +source = "git+https://github.com/apache/datafusion?rev=3291e4e#3291e4ec26a0efe361d8e4ddde56f9a27f8817ee" dependencies = [ "arrow", "bigdecimal", - "datafusion-common", - "datafusion-expr", + "datafusion-common 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", + "datafusion-expr 48.0.0 (git+https://github.com/apache/datafusion?rev=3291e4e)", "indexmap", "log", "regex", @@ -2673,6 +2885,17 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -3306,6 +3529,7 @@ dependencies = [ "num-bigint", "object_store", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -4522,17 +4746,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", diff --git a/native/Cargo.toml b/native/Cargo.toml index 54e459646c..2596e575f7 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,8 +38,8 @@ arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.10.0" } parquet = { version = "55.1.0", default-features = false, features = ["experimental"] } -datafusion = { version = "48.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-spark = { version = "48.0.0" } +datafusion = { git = "https://github.com/apache/datafusion", rev = "3291e4e", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "3291e4e" } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-proto = { path = "proto" } chrono = { version = "0.4", default-features = false, features = ["clock"] } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index e248068b35..dce5afa3ec 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -41,7 +41,7 @@ futures = { workspace = true } mimalloc = { version = "*", default-features = false, optional = true } tikv-jemallocator = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls"] } tikv-jemalloc-ctl = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls", "stats"] } -tokio = { version = "1", features = ["rt-multi-thread"] } +tokio = { version = "1.46.1", features = ["rt-multi-thread"] } async-trait = { workspace = true } log = "0.4" log4rs = "1.2.0" diff --git a/native/core/benches/shuffle_writer.rs b/native/core/benches/shuffle_writer.rs index ee07462d00..52638d92a9 100644 --- a/native/core/benches/shuffle_writer.rs +++ b/native/core/benches/shuffle_writer.rs @@ -89,7 +89,8 @@ fn criterion_benchmark(c: &mut Criterion) { CometPartitioning::RangePartitioning( LexOrdering::new(vec![PhysicalSortExpr::new_default( col("c0", batch.schema().as_ref()).unwrap(), - )]), + )]) + .unwrap(), 16, 100, ), diff --git a/native/core/src/execution/operators/filter.rs b/native/core/src/execution/operators/filter.rs index 573287a088..c8fcda19fe 100644 --- a/native/core/src/execution/operators/filter.rs +++ b/native/core/src/execution/operators/filter.rs @@ -211,22 +211,16 @@ impl FilterExec { if let Some(binary) = conjunction.as_any().downcast_ref::() { if binary.op() == &Operator::Eq { // Filter evaluates to single value for all partitions - if input_eqs.is_expr_constant(binary.left()) { - let (expr, across_parts) = ( - binary.right(), - input_eqs.get_expr_constant_value(binary.right()), - ); - res_constants.push( - ConstExpr::new(Arc::clone(expr)).with_across_partitions(across_parts), - ); - } else if input_eqs.is_expr_constant(binary.right()) { - let (expr, across_parts) = ( - binary.left(), - input_eqs.get_expr_constant_value(binary.left()), - ); - res_constants.push( - ConstExpr::new(Arc::clone(expr)).with_across_partitions(across_parts), - ); + if input_eqs.is_expr_constant(binary.left()).is_some() { + let across = input_eqs + .is_expr_constant(binary.right()) + .unwrap_or_default(); + res_constants.push(ConstExpr::new(Arc::clone(binary.right()), across)); + } else if input_eqs.is_expr_constant(binary.right()).is_some() { + let across = input_eqs + .is_expr_constant(binary.left()) + .unwrap_or_default(); + res_constants.push(ConstExpr::new(Arc::clone(binary.left()), across)); } } } @@ -246,7 +240,7 @@ impl FilterExec { let mut eq_properties = input.equivalence_properties().clone(); let (equal_pairs, _) = collect_columns_from_predicate(predicate); for (lhs, rhs) in equal_pairs { - eq_properties.add_equal_conditions(lhs, rhs)? + eq_properties.add_equal_conditions(Arc::clone(lhs), Arc::clone(rhs))?; } // Add the columns that have only one viable value (singleton) after // filtering to constants. @@ -258,14 +252,13 @@ impl FilterExec { .min_value .get_value(); let expr = Arc::new(column) as _; - ConstExpr::new(expr) - .with_across_partitions(AcrossPartitions::Uniform(value.cloned())) + ConstExpr::new(expr, AcrossPartitions::Uniform(value.cloned())) }); // This is for statistics - eq_properties = eq_properties.with_constants(constants); + eq_properties.add_constants(constants)?; // This is for logical constant (for example: a = '1', then a could be marked as a constant) // to do: how to deal with multiple situation to represent = (for example c1 between 0 and 0) - eq_properties = eq_properties.with_constants(Self::extend_constants(input, predicate)); + eq_properties.add_constants(Self::extend_constants(input, predicate))?; let mut output_partitioning = input.output_partitioning().clone(); // If contains projection, update the PlanProperties. diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index a45036be45..afb9c61d19 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -75,7 +75,7 @@ use crate::parquet::parquet_support::prepare_object_store_with_configs; use datafusion::common::scalar::ScalarStructBuilder; use datafusion::common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, - JoinType as DFJoinType, ScalarValue, + JoinType as DFJoinType, NullEquality, ScalarValue, }; use datafusion::datasource::listing::PartitionedFile; use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression; @@ -1115,7 +1115,7 @@ impl PhysicalPlanner { let child_copied = Self::wrap_in_copy_exec(Arc::clone(&child.native_plan)); let sort = Arc::new( - SortExec::new(LexOrdering::new(exprs?), Arc::clone(&child_copied)) + SortExec::new(LexOrdering::new(exprs?).unwrap(), Arc::clone(&child_copied)) .with_fetch(fetch), ); @@ -1391,7 +1391,7 @@ impl PhysicalPlanner { sort_options, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - false, + NullEquality::NullEqualsNothing, )?); if join.filter.is_some() { @@ -1459,7 +1459,7 @@ impl PhysicalPlanner { PartitionMode::Partitioned, // null doesn't equal to null in Spark join key. If the join key is // `EqualNullSafe`, Spark will rewrite it during planning. - false, + NullEquality::NullEqualsNothing, )?); // If the hash join is build right, we need to swap the left and right @@ -2163,7 +2163,7 @@ impl PhysicalPlanner { window_func_name, &window_args, partition_by, - &LexOrdering::new(sort_exprs.to_vec()), + &LexOrdering::new(sort_exprs.to_vec()).unwrap(), window_frame.into(), input_schema.as_ref(), false, // TODO: Ignore nulls @@ -2244,7 +2244,7 @@ impl PhysicalPlanner { .iter() .map(|expr| self.create_sort_expr(expr, Arc::clone(&input_schema))) .collect(); - let lex_ordering = LexOrdering::from(exprs?); + let lex_ordering = LexOrdering::new(exprs?).unwrap(); Ok(CometPartitioning::RangePartitioning( lex_ordering, range_partition.num_partitions as usize, @@ -2611,6 +2611,7 @@ mod tests { FileGroup, FileScanConfigBuilder, FileSource, ParquetSource, }; use datafusion::error::DataFusionError; + use datafusion::functions_nested::make_array::MakeArray; use datafusion::logical_expr::ScalarUDF; use datafusion::physical_plan::ExecutionPlan; use datafusion::{assert_batches_eq, physical_plan::common::collect, prelude::SessionContext}; @@ -2940,9 +2941,7 @@ mod tests { #[test] fn test_create_array() { let session_ctx = SessionContext::new(); - session_ctx.register_udf(ScalarUDF::from( - datafusion_functions_nested::make_array::MakeArray::new(), - )); + session_ctx.register_udf(ScalarUDF::from(MakeArray::new())); let task_ctx = session_ctx.task_ctx(); let planner = PhysicalPlanner::new(Arc::from(session_ctx), 0); diff --git a/native/core/src/execution/shuffle/range_partitioner.rs b/native/core/src/execution/shuffle/range_partitioner.rs index 8f443232f9..75dcc4bff9 100644 --- a/native/core/src/execution/shuffle/range_partitioner.rs +++ b/native/core/src/execution/shuffle/range_partitioner.rs @@ -243,7 +243,8 @@ mod test { let lex_ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default( col("a", input_batch.schema().as_ref()).unwrap(), - )]); + )]) + .unwrap(); let (rows, row_converter) = RangePartitioner::generate_bounds( input_batch.columns().to_vec().as_ref(), diff --git a/native/core/src/execution/shuffle/shuffle_writer.rs b/native/core/src/execution/shuffle/shuffle_writer.rs index d23d02a2c8..4b712c48bd 100644 --- a/native/core/src/execution/shuffle/shuffle_writer.rs +++ b/native/core/src/execution/shuffle/shuffle_writer.rs @@ -934,7 +934,7 @@ impl SinglePartitionShufflePartitioner { Ok(Some(concatenated)) } Err(e) => Err(DataFusionError::ArrowError( - e, + Box::new(e), Some(DataFusionError::get_back_trace()), )), } @@ -1122,7 +1122,7 @@ impl Iterator for PartitionedBatchIterator<'_> { Some(Ok(batch)) } Err(e) => Some(Err(DataFusionError::ArrowError( - e, + Box::new(e), Some(DataFusionError::get_back_trace()), ))), } @@ -1409,7 +1409,8 @@ mod test { CometPartitioning::RangePartitioning( LexOrdering::new(vec![PhysicalSortExpr::new_default( col("a", batch.schema().as_ref()).unwrap(), - )]), + )]) + .unwrap(), num_partitions, 100, ),