chore: Upgrade opentelemetry and remove metrics crate (#15484)

Signed-off-by: Xuanwo <github@xuanwo.io>

Author: Xuanwo

Cargo.lock

@@ -164,11 +164,7 @@ geo = { version = "0.27.0", features = ["use-serde"] }
 geos = { version = "8.3", features = ["static", "geo", "geo-types"] }
 geozero = { version = "0.11.0", features = ["default", "with-wkb", "with-geos", "with-geojson"] }
 itertools = "0.10.5"
-log = { version = "0.4.21", features = ["serde", "kv_unstable_std"] }
-logcall = "0.1.5"
 match-template = "0.0.1"
-metrics = "0.20.1"
-minitrace = { version = "0.6", features = ["enable"] }
 mysql_async = { version = "0.33", default-features = false, features = ["rustls-tls"] }
 ordered-float = { version = "4.1.0", default-features = false }
 poem = { version = "~1.3.57", features = ["rustls", "multipart", "compression"] }
@@ -242,6 +238,20 @@ tonic-build = { version = "0.10.2" }
 bumpalo = "3.12.0"
 tikv-jemalloc-ctl = { version = "0.5.0", features = ["use_std"] }
 
+# Observability
+log = { version = "0.4.21", features = ["serde", "kv_unstable_std"] }
+logcall = "0.1.5"
+minitrace = { version = "0.6", features = ["enable"] }
+minitrace-opentelemetry = "0.6"
+opentelemetry = { version = "0.22", features = ["trace", "logs"] }
+opentelemetry-otlp = { version = "0.15", features = ["trace", "logs", "grpc-tonic"] }
+opentelemetry_sdk = { version = "0.22", features = ["trace", "logs", "rt-tokio"] }
+tracing = "0.1.40"
+opentelemetry-http = "0.11.0"
+tracing-appender = "0.2.3"
+console-subscriber = { version = "0.2.0" }
+tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"] }
+
 [profile.release]
 debug = 1
 lto = "thin"
@@ -283,15 +293,16 @@ debug-assertions = true
 overflow-checks = true
 rpath = false
 
-[patch.crates-io]
 # If there are dependencies that need patching, they can be listed below.
-# Changes to upstream: update to tonic 0.11
+[patch.crates-io]
 arrow-format = { git = "https://github.com/Xuanwo/arrow-format", rev = "be633a0" }
-metrics = { git = "https://github.com/datafuse-extras/metrics.git", rev = "fc2ecd1" }
 icelake = { git = "https://github.com/icelake-io/icelake", rev = "be8b2c2" }
 micromarshal = { git = "https://github.com/ariesdevil/opensrv", rev = "6c96813" }
 async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" }
 z3 = { git = "https://github.com/prove-rs/z3.rs", rev = "247d308" }
 z3-sys = { git = "https://github.com/prove-rs/z3.rs", rev = "247d308" }
 geozero = { git = "https://github.com/georust/geozero", rev = "1d78b36" }
+# Wait for upstream to release new version.
+minitrace = { package = "minitrace", git = "https://github.com/tikv/minitrace-rust", rev = "b987f64" }
+minitrace-opentelemetry = { package = "minitrace-opentelemetry", git = "https://github.com/tikv/minitrace-rust", rev = "b987f64" }
 # proj = { git = "https://github.com/ariesdevil/proj", rev = "51e1c60" }

Cargo.toml

@@ -10,15 +10,11 @@ edition = { workspace = true }
 doctest = false
 test = true
 
-[features]
-enable-histogram = ["metrics/enable-histogram"]
-
 [dependencies]
 # Workspace dependencies
 databend-common-base = { path = "../base" }
 
 # Crates.io dependencies
-metrics = { workspace = true }
 prometheus-client = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]

src/common/metrics/Cargo.toml

@@ -13,10 +13,13 @@
 // limitations under the License.
 
 use std::sync::LazyLock;
+use std::time::Duration;
 
 use databend_common_base::runtime::metrics::register_counter;
+use databend_common_base::runtime::metrics::register_gauge;
 use databend_common_base::runtime::metrics::register_histogram_in_milliseconds;
 use databend_common_base::runtime::metrics::Counter;
+use databend_common_base::runtime::metrics::Gauge;
 use databend_common_base::runtime::metrics::Histogram;
 
 // Common metrics.
@@ -172,6 +175,9 @@ static COMPACT_BLOCK_BUILD_TASK_MILLISECONDS: LazyLock<Histogram> = LazyLock::ne
 static COMPACT_BLOCK_BUILD_LAZY_PART_MILLISECONDS: LazyLock<Histogram> = LazyLock::new(|| {
     register_histogram_in_milliseconds("fuse_compact_block_build_lazy_part_milliseconds")
 });
+static COMPACT_SEGMENTS_SELECT_DURATION_SECOND: LazyLock<Gauge> =
+    LazyLock::new(|| register_gauge("fuse_compact_segments_select_duration_second"));
+
 static RECLUSTER_BUILD_TASK_MILLISECONDS: LazyLock<Histogram> =
     LazyLock::new(|| register_histogram_in_milliseconds("fuse_recluster_build_task_milliseconds"));
 static RECLUSTER_SEGMENT_NUMS_SCHEDULED: LazyLock<Counter> =
@@ -561,6 +567,10 @@ pub fn metrics_inc_compact_block_build_lazy_part_milliseconds(c: u64) {
     COMPACT_BLOCK_BUILD_LAZY_PART_MILLISECONDS.observe(c as f64);
 }
 
+pub fn metrics_set_compact_segments_select_duration_second(c: Duration) {
+    COMPACT_SEGMENTS_SELECT_DURATION_SECOND.set(c.as_secs() as i64);
+}
+
 /// Pruning metrics.
 pub fn metrics_inc_segments_range_pruning_before(c: u64) {
     SEGMENTS_RANGE_PRUNING_BEFORE.inc_by(c);

src/common/metrics/src/metrics/storage.rs

@@ -26,7 +26,6 @@ dashmap = { workspace = true, features = ["serde"] }
 flagset = "0.4"
 futures = { workspace = true }
 log = { workspace = true }
-metrics = "0.20.1"
 opendal = { workspace = true }
 ordered-float = { workspace = true }
 parquet = { workspace = true }

src/common/storage/Cargo.toml

@@ -18,20 +18,20 @@ console = ["console-subscriber", "tracing", "tracing-subscriber"]
 databend-common-base = { path = "../base" }
 
 # Crates.io dependencies
-console-subscriber = { version = "0.2.0", optional = true }
+console-subscriber = { workspace = true, optional = true }
 defer = "0.2"
 fern = "0.6.2"
 humantime = "2.1.0"
 itertools = { workspace = true }
 log = { workspace = true }
 minitrace = { workspace = true }
-minitrace-opentelemetry = "0.6"
-opentelemetry = { version = "0.21", features = ["trace", "logs"] }
-opentelemetry-otlp = { version = "0.14", features = ["trace", "logs", "grpc-tonic"] }
-opentelemetry_sdk = { version = "0.21", features = ["trace", "logs", "rt-tokio"] }
+minitrace-opentelemetry = { workspace = true }
+opentelemetry = { workspace = true }
+opentelemetry-otlp = { workspace = true }
+opentelemetry_sdk = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tonic = { workspace = true }
-tracing = { version = "0.1.40", optional = true }
-tracing-appender = "0.2.3"
-tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"], optional = true }
+tracing = { workspace = true, optional = true }
+tracing-appender = { workspace = true }
+tracing-subscriber = { workspace = true, optional = true }

src/common/tracing/Cargo.toml

@@ -13,9 +13,6 @@ edition = { workspace = true }
 doctest = false
 test = true
 
-[features]
-enable-histogram = ["databend-common-metrics/enable-histogram"]
-
 [dependencies]
 databend-common-arrow = { path = "../../common/arrow" }
 databend-common-base = { path = "../../common/base" }