From 37942a54df85e58105d12bd1f21c5dbaf0a031d2 Mon Sep 17 00:00:00 2001 From: coldWater Date: Fri, 18 Jul 2025 12:13:35 +0800 Subject: [PATCH] x --- Cargo.lock | 12 +++++- Cargo.toml | 2 + src/query/config/Cargo.toml | 2 + src/query/config/src/config.rs | 4 +- src/query/config/src/env.rs | 75 ++++++++++++++++++++++++++++++++++ src/query/config/src/lib.rs | 1 + 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 src/query/config/src/env.rs diff --git a/Cargo.lock b/Cargo.lock index d95a92b13bef2..6833775a85040 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,6 +3269,8 @@ dependencies = [ "log", "pretty_assertions", "serde", + "serde-bridge", + "serde-env 0.2.0", "serde_ignored", "serde_with", "serfig", @@ -13644,6 +13646,14 @@ dependencies = [ "serde", ] +[[package]] +name = "serde-env" +version = "0.2.0" +source = "git+https://github.com/Xuanwo/serde-env?rev=085d4b1#085d4b1846ad16e3b7062bf11bf40a1c38d746e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.17" @@ -13800,7 +13810,7 @@ dependencies = [ "log", "serde", "serde-bridge", - "serde-env", + "serde-env 0.1.1", "toml 0.7.8", ] diff --git a/Cargo.toml b/Cargo.toml index 077ac5559d7b1..0919b8ea63b62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -478,6 +478,8 @@ self_cell = "1.2.0" semver = "1.0.14" seq-marked = { version = "0.3.1", features = ["seq-marked-serde", "seq-marked-bincode", "seqv-serde"] } serde = { version = "1.0.164", features = ["derive", "rc"] } +serde-bridge = "0.0.3" +serde-env = { git = "https://github.com/Xuanwo/serde-env", rev = "085d4b1" } serde_derive = "1" serde_ignored = "0.1.10" serde_json = { version = "1.0.85", default-features = false, features = ["preserve_order", "unbounded_depth"] } diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index bb763182ad8a9..7b06f094e1cd5 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -27,6 +27,8 @@ databend-common-tracing = { workspace = true } databend-common-version = { workspace = true } log = { workspace = true } serde = { workspace = true } +serde-bridge = { workspace = true } +serde-env = { workspace = true } serde_ignored = { workspace = true } serde_with = { workspace = true } serfig = { workspace = true } diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 7a4329633b86d..39f173f1ba142 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -61,10 +61,10 @@ use databend_common_version::DATABEND_COMMIT_VERSION; use serde::Deserialize; use serde::Serialize; use serde_with::with_prefix; -use serfig::collectors::from_env; use serfig::collectors::from_file; use serfig::collectors::from_self; +use super::env::from_env; use super::inner; use super::inner::CatalogConfig as InnerCatalogConfig; use super::inner::CatalogHiveConfig as InnerCatalogHiveConfig; @@ -3520,7 +3520,7 @@ pub struct SpillConfig { /// Allow space in bytes to spill to local disk. pub spill_local_disk_max_bytes: u64, - // TODO: We need to fix StorageConfig so that it supports environment variables and command line injections. + // TODO: We need to fix StorageConfig so that it supports command line injections. #[clap(skip)] pub storage: Option, } diff --git a/src/query/config/src/env.rs b/src/query/config/src/env.rs new file mode 100644 index 0000000000000..ed37b0db6a3bd --- /dev/null +++ b/src/query/config/src/env.rs @@ -0,0 +1,75 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::marker::PhantomData; + +use log::debug; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_bridge::IntoValue; +use serde_bridge::Value; +use serfig::collectors::IntoCollector; +use serfig::Collector; + +pub fn from_env() -> Environment +where V: DeserializeOwned + Serialize + Debug { + Environment { + _v: Default::default(), + } +} + +#[derive(Debug)] +pub struct Environment { + _v: PhantomData, +} + +impl Collector for Environment +where V: DeserializeOwned + Serialize + Debug +{ + fn collect(&mut self) -> anyhow::Result { + let v: V = serde_env::from_env()?; + debug!("value parsed from env: {:?}", v); + Ok(v.into_value()?) + } +} + +impl IntoCollector for Environment +where V: DeserializeOwned + Serialize + Debug + 'static +{ + fn into_collector(self) -> Box> { + Box::new(self) + } +} + +#[cfg(test)] +mod tests { + use crate::Config; + + #[test] + fn test_env() { + // see https://github.com/Xuanwo/serde-env/pull/48 + let cfg: Config = serde_env::from_iter([ + ("LOG_TRACING_OTLP_ENDPOINT", "http://127.0.2.1:1111"), + ("LOG_TRACING_CAPTURE_LOG_LEVEL", "DebuG"), + ]) + .unwrap(); + + assert_eq!( + cfg.log.tracing.tracing_otlp.endpoint, + "http://127.0.2.1:1111" + ); + assert_eq!(cfg.log.tracing.tracing_capture_log_level, "DebuG"); + } +} diff --git a/src/query/config/src/lib.rs b/src/query/config/src/lib.rs index 2889737e4c411..5563e0a47938c 100644 --- a/src/query/config/src/lib.rs +++ b/src/query/config/src/lib.rs @@ -29,6 +29,7 @@ mod builtin; /// - `TryInto for config::Config` /// - `From for config::Config` mod config; +mod env; mod global; mod inner; mod mask;