Merge pull request #9190 from BohuTANG/dev-refine-string

refactor: move string func from base/io to base/string_func

Author: BohuTANG

Cargo.lock

@@ -30,6 +30,7 @@ async-channel = "1.7.1"
 async-trait = "0.1.57"
 bytesize = "1.1.0"
 ctrlc = { version = "3.2.3", features = ["termination"] }
+enquote = "1.1.0"
 futures = "0.3.24"
 libc = "0.2.133"
 num_cpus = "1.13.1"

src/common/base/Cargo.toml

@@ -24,7 +24,7 @@ mod shutdown_signal;
 mod singleton_instance;
 mod stop_handle;
 mod stoppable;
-mod string_func;
+mod string;
 mod thread;
 mod thread_pool;
 mod uniq_id;
@@ -52,10 +52,13 @@ pub use shutdown_signal::SignalType;
 pub use singleton_instance::GlobalInstance;
 pub use stop_handle::StopHandle;
 pub use stoppable::Stoppable;
-pub use string_func::escape_for_key;
-pub use string_func::mask_string;
-pub use string_func::replace_nth_char;
-pub use string_func::unescape_for_key;
+pub use string::convert_byte_size;
+pub use string::convert_number_size;
+pub use string::escape_for_key;
+pub use string::mask_string;
+pub use string::replace_nth_char;
+pub use string::unescape_for_key;
+pub use string::unescape_string;
 pub use thread::Thread;
 pub use thread::ThreadJoinHandle;
 pub use thread_pool::TaskJoinHandler;

src/common/base/src/base/mod.rs

@@ -14,6 +14,9 @@
 
 use std::string::FromUtf8Error;
 
+use common_exception::ErrorCode;
+use common_exception::Result;
+
 /// Function that escapes special characters in a string.
 ///
 /// All characters except digit, alphabet and '_' are treated as special characters.
@@ -114,3 +117,55 @@ pub fn replace_nth_char(s: &str, idx: usize, newchar: char) -> String {
         .map(|(i, c)| if i == idx { newchar } else { c })
         .collect()
 }
+
+/// Returns string after processing escapes.
+/// This used for settings string unescape, like unescape format_field_delimiter from `\\x01` to `\x01`.
+pub fn unescape_string(escape_str: &str) -> Result<String> {
+    enquote::unescape(escape_str, None)
+        .map_err(|e| ErrorCode::Internal(format!("unescape:{} error:{:?}", escape_str, e)))
+}
+
+pub fn convert_byte_size(num: f64) -> String {
+    let negative = if num.is_sign_positive() { "" } else { "-" };
+    let num = num.abs();
+    let units = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB"];
+    if num < 1_f64 {
+        return format!("{}{:.02} {}", negative, num, "B");
+    }
+    let delimiter = 1024_f64;
+    let exponent = std::cmp::min(
+        (num.ln() / delimiter.ln()).floor() as i32,
+        (units.len() - 1) as i32,
+    );
+    let pretty_bytes = format!("{:.02}", num / delimiter.powi(exponent));
+    let unit = units[exponent as usize];
+    format!("{}{} {}", negative, pretty_bytes, unit)
+}
+
+pub fn convert_number_size(num: f64) -> String {
+    let negative = if num.is_sign_positive() { "" } else { "-" };
+    let num = num.abs();
+    let units = [
+        "",
+        " thousand",
+        " million",
+        " billion",
+        " trillion",
+        " quadrillion",
+    ];
+
+    if num < 1_f64 {
+        return format!("{}{}", negative, num);
+    }
+    let delimiter = 1000_f64;
+    let exponent = std::cmp::min(
+        (num.ln() / delimiter.ln()).floor() as i32,
+        (units.len() - 1) as i32,
+    );
+    let pretty_bytes = format!("{:.2}", num / delimiter.powi(exponent))
+        .parse::<f64>()
+        .unwrap()
+        * 1_f64;
+    let unit = units[exponent as usize];
+    format!("{}{}{}", negative, pretty_bytes, unit)
+}

src/common/base/src/base/string_func.rs renamed to src/common/base/src/base/string.rs

@@ -42,3 +42,42 @@ fn replace_nth_char_test() {
     assert_eq!("a23".to_string(), replace_nth_char("a13", 1, '2'));
     assert_eq!("a13".to_string(), replace_nth_char("a13", 10, '2'));
 }
+
+#[test]
+fn convert_test() {
+    assert_eq!(convert_byte_size(0_f64), "0.00 B");
+    assert_eq!(convert_byte_size(0.1_f64), "0.10 B");
+    assert_eq!(convert_byte_size(1_f64), "1.00 B");
+    assert_eq!(convert_byte_size(1023_f64), "1023.00 B");
+    assert_eq!(convert_byte_size(1024_f64), "1.00 KiB");
+    assert_eq!(convert_byte_size(1229_f64), "1.20 KiB");
+    assert_eq!(
+        convert_byte_size(1024_f64 * 1024_f64 * 1024_f64),
+        "1.00 GiB"
+    );
+
+    assert_eq!(convert_number_size(1_f64), "1");
+    assert_eq!(convert_number_size(1022_f64), "1.02 thousand");
+    assert_eq!(convert_number_size(10222_f64), "10.22 thousand");
+}
+
+#[test]
+fn test_unescape_string() {
+    let cases = vec![
+        vec!["a", "a"],
+        vec!["abc", "abc"],
+        vec!["\\x01", "\x01"],
+        vec!["\x01", "\x01"],
+        vec!["\t\nabc", "\t\nabc"],
+        vec!["\"\t\nabc\"", "\"\t\nabc\""],
+        vec!["\"\\t\nabc\"", "\"\t\nabc\""],
+        vec!["'\\t\nabc'", "'\t\nabc'"],
+        vec!["\\t\\nabc", "\t\nabc"],
+        vec!["\\\\", r"\"],
+        vec!["\\\\", "\\"],
+    ];
+
+    for c in cases {
+        assert_eq!(unescape_string(c[0]).unwrap(), c[1]);
+    }
+}

src/common/base/tests/it/string_func.rs

@@ -21,7 +21,6 @@ bincode = { version = "2.0.0-rc.1", features = ["serde", "std"] }
 bytes = "1.2.1"
 chrono = { workspace = true }
 chrono-tz = { workspace = true }
-enquote = "1.1.0"
 lexical-core = "0.8.5"
 micromarshal = "0.2.1"
 ordered-float = "3.1.0"

src/common/io/Cargo.toml

@@ -35,5 +35,5 @@ mod file_split;
 mod format_settings;
 mod options_deserializer;
 mod position;
+mod serialization;
 mod stat_buffer;
-mod utils;

src/common/io/src/lib.rs

@@ -31,5 +31,5 @@ pub use crate::format_settings::FormatSettings;
 pub use crate::options_deserializer::OptionsDeserializer;
 pub use crate::options_deserializer::OptionsDeserializerError;
 pub use crate::position::*;
+pub use crate::serialization::*;
 pub use crate::stat_buffer::StatBuffer;
-pub use crate::utils::*;

src/common/io/src/prelude.rs

@@ -0,0 +1,34 @@
+// Copyright 2021 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_exception::Result;
+
+/// bincode seralize_into wrap with optimized config
+#[inline]
+pub fn serialize_into_buf<W: std::io::Write, T: serde::Serialize>(
+    writer: &mut W,
+    value: &T,
+) -> Result<()> {
+    bincode::serde::encode_into_std_write(value, writer, bincode::config::standard())?;
+    Ok(())
+}
+
+/// bincode deserialize_from wrap with optimized config
+#[inline]
+pub fn deserialize_from_slice<T: serde::de::DeserializeOwned>(slice: &mut &[u8]) -> Result<T> {
+    let (value, bytes_read) =
+        bincode::serde::decode_from_slice(slice, bincode::config::standard())?;
+    *slice = &slice[bytes_read..];
+    Ok(value)
+}