feat: custom csv record delimiter

Author: BohuTANG

src/query/formats/src/format_option_checker.rs

@@ -230,17 +230,25 @@ pub fn check_field_delimiter(option: &mut String, default: &str) -> Result<()> {
     Ok(())
 }
 
+/// `\r\n` or u8
 pub fn check_record_delimiter(option: &mut String) -> Result<()> {
-    if option.is_empty() {
-        *option = "\n".to_string()
-    } else {
-        let o = option.as_str();
-        if o != "\n" && o != "\r\n" {
+    match option.len() {
+        0 => *option = "\n".to_string(),
+        1 => {}
+        2 => {
+            if option != "\r\n" {
+                return Err(ErrorCode::InvalidArgument(
+                    "record_delimiter with two chars can only be '\\r\\n'",
+                ));
+            };
+        }
+        _ => {
             return Err(ErrorCode::InvalidArgument(
-                "record_delimiter can only be '\\n' or '\\r\\n'",
+                "record_delimiter can not more than two chars, please use one char or '\\r\\n'",
             ));
-        };
+        }
     }
+
     Ok(())
 }
 

tests/suites/1_stateful/05_formats/05_02_csv/05_02_04_csv_no_newline.sh

@@ -11,6 +11,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
      b Int NULL
  );" | $MYSQL_CLIENT_CONNECT
 
- curl -sH "insert_sql:insert into test_csv format CSV" -F "upload=@${CURDIR}/no_newline.csv" \
+ curl -sH "insert_sql:insert into test_csv format CSV" -F "upload=@${CURDIR}/testdata/no_newline.csv" \
  -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"
  echo "select * from test_csv" | $MYSQL_CLIENT_CONNECT

tests/suites/1_stateful/05_formats/05_02_csv/05_02_05_csv_with_0x01_delimiter.result renamed to tests/suites/1_stateful/05_formats/05_02_csv/05_02_05_csv_0x01_field_0x02_record.result

@@ -12,6 +12,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
      c VARCHAR
  );" | $MYSQL_CLIENT_CONNECT
 
- curl -sH "insert_sql:insert into test_x01_csv format CSV" -H "format_field_delimiter:\x01" -H "format_skip_header:1" -H "format_record_delimiter:\r\n" -F "upload=@${CURDIR}/x01_field_delimiter.csv" \
+ curl -sH "insert_sql:insert into test_x01_csv format CSV" -H "format_field_delimiter:\x01" -H "format_record_delimiter:\x02" -H "format_skip_header:0" -F "upload=@${CURDIR}/testdata/x01_field_x02_record.csv" \
  -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"
  echo "select count() from test_x01_csv" | $MYSQL_CLIENT_CONNECT

tests/suites/1_stateful/05_formats/05_02_csv/05_02_05_csv_with_0x01_delimiter.sh renamed to tests/suites/1_stateful/05_formats/05_02_csv/05_02_05_csv_0x01_field_0x02_record.sh

@@ -0,0 +1 @@
+"Donald John Trump"
80
"president""马斯克"
43
"特斯拉""测试"
39
"TT""测试"
39
"test"