feat: Upgrade OpenDAL to v0.16.0

Signed-off-by: Xuanwo <github@xuanwo.io>

Author: Xuanwo

Cargo.lock

@@ -12,4 +12,4 @@ test = false
 common-base = { path = "../base" }
 
 async-trait = "0.1.56"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }

src/common/contexts/Cargo.toml

@@ -18,9 +18,8 @@ anyhow = "1.0.58"
 backon = "0.0.2"
 globiter = "0.1.0"
 once_cell = "1.12.0"
-opendal = { version = "0.14.1", features = [
+opendal = { version = "0.16.0", features = [
     "layers-retry",
-    "services-http",
     "layers-tracing",
     "layers-metrics",
     "compress",

src/common/storage/Cargo.toml

@@ -21,6 +21,7 @@ use common_base::base::Singleton;
 use common_contexts::DalRuntime;
 use common_exception::ErrorCode;
 use once_cell::sync::OnceCell;
+use opendal::layers::ImmutableIndexLayer;
 use opendal::layers::LoggingLayer;
 use opendal::layers::MetricsLayer;
 use opendal::layers::RetryLayer;
@@ -127,10 +128,16 @@ pub fn init_http_operator(cfg: &StorageHttpConfig) -> Result<Operator> {
     // Endpoint.
     builder.endpoint(&cfg.endpoint_url);
 
-    // Update index.
-    builder.extend_index(cfg.paths.iter().map(|v| v.as_str()));
+    // HTTP Service is read-only and doesn't support list operation.
+    // ImmutableIndexLayer will build an in-memory immutable index for it.
+    let mut immutable_layer = ImmutableIndexLayer::default();
+    let files: Vec<String> = cfg.paths.iter().map(|v| v.to_string()).collect();
+    // TODO: should be replace by `immutable_layer.extend_iter()` after fix
+    for i in files {
+        immutable_layer.insert(i);
+    }
 
-    Ok(Operator::new(builder.build()?))
+    Ok(Operator::new(builder.build()?).layer(immutable_layer))
 }
 
 /// init_memory_operator will init a opendal memory operator.

src/common/storage/src/operator.rs

@@ -26,5 +26,5 @@ common-users = { path = "../users" }
 
 async-trait = "0.1.56"
 dyn-clone = "1.0.6"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }
 parking_lot = "0.12.1"

src/query/catalog/Cargo.toml

@@ -21,7 +21,7 @@ common-users = { path = "../users" }
 clap = { version = "3.2.5", features = ["derive", "env"] }
 hex = "0.4.3"
 once_cell = "1.12.0"
-opendal = { version = "0.14.1", features = ["layers-retry", "compress"], optional = true }
+opendal = { version = "0.16.0", features = ["layers-retry", "compress"], optional = true }
 semver = "1.0.10"
 serde = { version = "1.0.137", features = ["derive"] }
 serfig = "0.0.2"

src/query/config/Cargo.toml

@@ -23,5 +23,5 @@ common-streams = { path = "../../streams" }
 async-trait = { version = "0.1.0", package = "async-trait-fn" }
 futures = "0.3.21"
 futures-util = "0.3.21"
-opendal = { version = "0.14.1", features = ["layers-retry", "compress"] }
+opendal = { version = "0.16.0", features = ["layers-retry", "compress"] }
 parking_lot = "0.12.1"

src/query/pipeline/sources/Cargo.toml

@@ -111,7 +111,7 @@ num = "0.4.0"
 num_cpus = "1.13.1"
 octocrab = "0.16.0"
 once_cell = "1.12.0"
-opendal = { version = "0.14.1", features = ["layers-retry", "layers-tracing", "layers-metrics", "compress"] }
+opendal = { version = "0.16.0", features = ["layers-retry", "layers-tracing", "layers-metrics", "compress"] }
 opensrv-mysql = "0.2.0"
 openssl = { version = "0.10.40", features = ["vendored"] }
 parking_lot = "0.12.1"

src/query/service/Cargo.toml

@@ -37,7 +37,7 @@ backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 chrono = "0.4.19"
 futures = "0.3.21"
 futures-util = "0.3.21"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }
 serde = { version = "1.0.137", features = ["derive"] }
 serde_json = "1.0.81"
 tracing = "0.1.35"

src/query/storages/fuse/Cargo.toml

@@ -31,7 +31,7 @@ common-storages-util = { path = "../util" }
 async-recursion = "1.0.0"
 async-trait = "0.1.56"
 futures = "0.3.21"
-opendal = { version = "0.14.1", features = ["layers-retry"] }
+opendal = { version = "0.16.0", features = ["layers-retry"] }
 serde = { version = "1.0.137", features = ["derive"] }
 thrift = "0.15.0"
 tracing = "0.1.35"