build(deps): upgrade datafusion from 0.41 to 0.42

.github/workflows/ci.yml

@@ -77,10 +77,10 @@ jobs:
       matrix:
         # TODO: add windows
         os: [ ubuntu-22.04, macos-14 ]
-        python-version: [ '3.8', '3.12' ]
+        python-version: [ '3.9', '3.12' ]
         exclude:
           - os: macos-14
-            python-version: '3.8'
+            python-version: '3.9'
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

Cargo.toml

@@ -35,25 +35,25 @@ repository = "https://github.com/apache/hudi-rs"
 
 [workspace.dependencies]
 # arrow
-arrow = { version = "= 52.2.0", features = ["pyarrow"] }
-arrow-arith = { version = "= 52.2.0" }
-arrow-array = { version = "= 52.2.0" }
-arrow-buffer = { version = "= 52.2.0" }
-arrow-cast = { version = "= 52.2.0" }
-arrow-ipc = { version = "= 52.2.0" }
-arrow-json = { version = "= 52.2.0" }
-arrow-ord = { version = "= 52.2.0" }
-arrow-row = { version = "= 52.2.0" }
-arrow-schema = { version = "= 52.2.0", features = ["serde"] }
-arrow-select = { version = "= 52.2.0" }
-object_store = { version = "= 0.10.2", features = ["aws", "azure", "gcp"] }
-parquet = { version = "= 52.2.0", features = ["async", "object_store"] }
+arrow = { version = "= 53.0.0", features = ["pyarrow"] }
+arrow-arith = { version = "= 53.0.0" }
+arrow-array = { version = "= 53.0.0" }
+arrow-buffer = { version = "= 53.0.0" }
+arrow-cast = { version = "= 53.0.0" }
+arrow-ipc = { version = "= 53.0.0" }
+arrow-json = { version = "= 53.0.0" }
+arrow-ord = { version = "= 53.0.0" }
+arrow-row = { version = "= 53.0.0" }
+arrow-schema = { version = "= 53.0.0", features = ["serde"] }
+arrow-select = { version = "= 53.0.0" }
+object_store = { version = "= 0.11.0", features = ["aws", "azure", "gcp"] }
+parquet = { version = "= 53.0.0", features = ["async", "object_store"] }
 
 # datafusion
-datafusion = { version = "= 41.0.0" }
-datafusion-expr = { version = "= 41.0.0" }
-datafusion-common = { version = "= 41.0.0" }
-datafusion-physical-expr = { version = "= 41.0.0" }
+datafusion = { version = "= 42.0.0" }
+datafusion-expr = { version = "= 42.0.0" }
+datafusion-common = { version = "= 42.0.0" }
+datafusion-physical-expr = { version = "= 42.0.0" }
 
 # serde
 serde = { version = "1.0.203", features = ["derive"] }

crates/datafusion/src/lib.rs

@@ -178,7 +178,7 @@ mod tests {
     ) -> SessionContext {
         let config = SessionConfig::new().set(
             "datafusion.sql_parser.enable_ident_normalization",
-            ScalarValue::from(false),
+            &ScalarValue::from(false),
         );
         let ctx = SessionContext::new_with_config(config);
         let base_url = test_table.url();
@@ -201,16 +201,16 @@ mod tests {
         let explaining_rb = explaining_rb.first().unwrap();
         let plan = get_str_column(explaining_rb, "plan").join("");
         let plan_lines: Vec<&str> = plan.lines().map(str::trim).collect();
-        assert!(plan_lines[2].starts_with("SortExec: TopK(fetch=10)"));
-        assert!(plan_lines[3].starts_with(&format!(
+        assert!(plan_lines[1].starts_with("SortExec: TopK(fetch=10)"));
+        assert!(plan_lines[2].starts_with(&format!(
             "ProjectionExec: expr=[id@0 as id, name@1 as name, isActive@2 as isActive, \
             get_field(structField@3, field2) as {}.structField[field2]]",
             table_name
         )));
-        assert!(plan_lines[5].starts_with(
+        assert!(plan_lines[4].starts_with(
             "FilterExec: CAST(id@0 AS Int64) % 2 = 0 AND get_field(structField@3, field2) > 30"
         ));
-        assert!(plan_lines[6].contains(&format!("input_partitions={}", planned_input_partitioned)));
+        assert!(plan_lines[5].contains(&format!("input_partitions={}", planned_input_partitioned)));
     }
 
     async fn verify_data(ctx: &SessionContext, sql: &str, table_name: &str) {

python/Cargo.toml

@@ -46,5 +46,5 @@ futures = { workspace = true }
 tokio = { workspace = true }
 
 [dependencies.pyo3]
-version = "0.21.2"
-features = ["extension-module", "abi3", "abi3-py38", "gil-refs", "anyhow"]
+version = "0.22.2"
+features = ["extension-module", "abi3", "abi3-py39", "anyhow"]

python/pyproject.toml

@@ -23,19 +23,18 @@ build-backend = "maturin"
 name = "hudi"
 description = "Native Python binding for Apache Hudi, based on hudi-rs."
 urls = { repository = "https://github.com/apache/hudi-rs/tree/main/python/" }
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 keywords = ["apachehudi", "hudi", "datalake", "arrow"]
 license = "Apache License 2.0"
 classifiers = [
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12"
 ]
 dependencies = [
-    "pyarrow>=8",
+    "pyarrow>=17",
     "pyarrow-hotfix",
 ]
 
@@ -52,7 +51,7 @@ dynamic = ["version"]
 module-name = "hudi._internal"
 
 [tool.ruff]
-target-version = 'py38'
+target-version = 'py39'
 # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
 lint.select = [
     "E4",