Merge branch 'main' into unwind-safe-operator

Author: mergify[bot]

Cargo.lock

@@ -25,7 +25,7 @@ Databend uses the latest techniques in vectorized query processing to allow you
 
 - __Easy to Use__
 
-  Databend has no indexes to build, no manual tuning required, no manual figuring out partitions or shard data, it’s all done for you as data is loaded into the table.
+  No manual tuning is required. You don't need to manually index, partition, or shard the data. Once you load your data into a table, Databend will handle everything else.
 
 ## Design Overview
 

docs/doc/00-overview/index.md

@@ -21,6 +21,7 @@ bincode = { version = "2.0.0-rc.1", features = ["serde", "std"] }
 bytes = "1.2.1"
 chrono = { workspace = true }
 chrono-tz = { workspace = true }
+enquote = "1.1.0"
 lexical-core = "0.8.5"
 micromarshal = "0.2.1"
 ordered-float = "3.1.0"

src/common/io/Cargo.toml

@@ -14,6 +14,7 @@
 
 use std::cmp;
 
+use common_exception::ErrorCode;
 use common_exception::Result;
 
 pub fn convert_byte_size(num: f64) -> String {
@@ -80,63 +81,11 @@ pub fn deserialize_from_slice<T: serde::de::DeserializeOwned>(slice: &mut &[u8])
     Ok(value)
 }
 
-pub fn is_control_ascii(c: u8) -> bool {
-    c <= 31
-}
-
-pub fn parse_escape_string(bs: &[u8]) -> String {
-    let bs = parse_escape_bytes(bs);
-
-    let mut cs = Vec::with_capacity(bs.len());
-    for b in bs {
-        cs.push(b as char);
-    }
-    cs.iter().collect()
-}
-
-pub fn parse_escape_bytes(bs: &[u8]) -> Vec<u8> {
-    let mut vs = Vec::with_capacity(bs.len());
-    let mut i = 0;
-    while i < bs.len() {
-        if bs[i] == b'\\' {
-            if i + 1 < bs.len() {
-                let c = parse_escape_byte(bs[i + 1]);
-                if c != b'\\'
-                    && c != b'\''
-                    && c != b'"'
-                    && c != b'`'
-                    && c != b'/'
-                    && !is_control_ascii(c)
-                {
-                    vs.push(b'\\');
-                }
-
-                vs.push(c);
-                i += 2;
-            } else {
-                // end with \
-                vs.push(b'\\');
-                break;
-            }
-        } else {
-            vs.push(bs[i]);
-            i += 1;
-        }
-    }
-
-    vs
-}
-
-// https://doc.rust-lang.org/reference/tokens.html
-pub fn parse_escape_byte(b: u8) -> u8 {
-    match b {
-        b'e' => b'\x1B',
-        b'n' => b'\n',
-        b'r' => b'\r',
-        b't' => b'\t',
-        b'0' => b'\0',
-        _ => b,
-    }
+/// Returns string after processing escapes.
+/// This used for settings string unescape, like unescape format_field_delimiter from `\\x01` to `\x01`.
+pub fn unescape_string(escape_str: &str) -> Result<String> {
+    enquote::unescape(escape_str, None)
+        .map_err(|e| ErrorCode::Internal(format!("unescape:{} error:{:?}", escape_str, e)))
 }
 
 /// Mask a string by "******", but keep `unmask_len` of suffix.

src/common/io/src/utils.rs

@@ -33,15 +33,22 @@ fn convert_test() {
 }
 
 #[test]
-fn parse_escape() {
+fn test_unescape_string() {
     let cases = vec![
         vec!["a", "a"],
         vec!["abc", "abc"],
+        vec!["\\x01", "\x01"],
+        vec!["\x01", "\x01"],
         vec!["\t\nabc", "\t\nabc"],
+        vec!["\"\t\nabc\"", "\"\t\nabc\""],
+        vec!["\"\\t\nabc\"", "\"\t\nabc\""],
+        vec!["'\\t\nabc'", "'\t\nabc'"],
         vec!["\\t\\nabc", "\t\nabc"],
+        vec!["\\\\", r"\"],
+        vec!["\\\\", "\\"],
     ];
 
     for c in cases {
-        assert_eq!(parse_escape_bytes(c[0].as_bytes()), c[1].as_bytes());
+        assert_eq!(unescape_string(c[0]).unwrap(), c[1]);
     }
 }

src/common/io/tests/it/utils.rs

@@ -230,17 +230,25 @@ pub fn check_field_delimiter(option: &mut String, default: &str) -> Result<()> {
     Ok(())
 }
 
+/// `\r\n` or u8
 pub fn check_record_delimiter(option: &mut String) -> Result<()> {
-    if option.is_empty() {
-        *option = "\n".to_string()
-    } else {
-        let o = option.as_str();
-        if o != "\n" && o != "\r\n" {
+    match option.len() {
+        0 => *option = "\n".to_string(),
+        1 => {}
+        2 => {
+            if option != "\r\n" {
+                return Err(ErrorCode::InvalidArgument(
+                    "record_delimiter with two chars can only be '\\r\\n'",
+                ));
+            };
+        }
+        _ => {
             return Err(ErrorCode::InvalidArgument(
-                "record_delimiter can only be '\\n' or '\\r\\n'",
+                "record_delimiter can not more than two chars, please use one char or '\\r\\n'",
             ));
-        };
+        }
     }
+
     Ok(())
 }
 

src/query/formats/src/format_option_checker.rs

@@ -28,5 +28,6 @@ pub use field_decoder::*;
 pub use file_format_type::parse_timezone;
 pub use file_format_type::FileFormatOptionsExt;
 pub use file_format_type::FileFormatTypeExt;
+pub use format_option_checker::check_record_delimiter;
 
 use crate::common_settings::CommonSettings;

src/query/formats/src/lib.rs

@@ -0,0 +1,37 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_formats::check_record_delimiter;
+
+/// This test code is written by OpenAI's GPT-3.
+#[test]
+fn test_check_record_delimiter() {
+    let mut option = "".to_string();
+    assert!(check_record_delimiter(&mut option).is_ok());
+    assert_eq!(option, "\n");
+
+    let mut option = "|".to_string();
+    assert!(check_record_delimiter(&mut option).is_ok());
+    assert_eq!(option, "|");
+
+    let mut option = "\r\n".to_string();
+    assert!(check_record_delimiter(&mut option).is_ok());
+    assert_eq!(option, "\r\n");
+
+    let mut option = "foo".to_string();
+    assert!(check_record_delimiter(&mut option).is_err());
+
+    let mut option = "|\r".to_string();
+    assert!(check_record_delimiter(&mut option).is_err());
+}

src/query/formats/tests/it/format_option_checker.rs

@@ -21,6 +21,7 @@ use common_meta_types::StageFileFormatType;
 use common_settings::Settings;
 
 mod field_encoder;
+mod format_option_checker;
 mod output_format_json_each_row;
 mod output_format_tcsv;
 mod output_format_utils;

src/query/formats/tests/it/main.rs

@@ -37,6 +37,7 @@ pub struct InputFormatTSV {}
 impl InputFormatTSV {
     #[allow(clippy::too_many_arguments)]
     fn read_row(
+        field_delimiter: u8,
         field_decoder: &FieldDecoderTSV,
         buf: &[u8],
         deserializers: &mut Vec<common_datavalues::TypeDeserializerImpl>,
@@ -53,7 +54,7 @@ impl InputFormatTSV {
         let mut err_msg = None;
         let buf_len = buf.len();
         while pos <= buf_len {
-            if pos == buf_len || buf[pos] == b'\t' {
+            if pos == buf_len || buf[pos] == field_delimiter {
                 let col_data = &buf[field_start..pos];
                 if col_data.is_empty() {
                     deserializers[column_index].de_default();
@@ -156,6 +157,7 @@ impl InputFormatTextBase for InputFormatTSV {
         for (i, end) in batch.row_ends.iter().enumerate() {
             let buf = &batch.data[start..*end]; // include \n
             Self::read_row(
+                builder.ctx.field_delimiter,
                 field_decoder,
                 buf,
                 columns,