Merge pull request #9195 from youngsofun/delimiter

fix(format): format option values are unescaped twice.

Author: BohuTANG

.github/workflows/dev-linux.yml

@@ -189,7 +189,7 @@ jobs:
       - uses: ./.github/actions/test_stateless_standalone_linux
 
   test_stateless_cluster_linux:
-    timeout-minutes: 20
+    timeout-minutes: 30
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu
     steps:

src/query/sql/src/planner/binder/copy.rs

@@ -24,7 +24,6 @@ use common_ast::parser::parse_sql;
 use common_ast::parser::tokenize_sql;
 use common_ast::Backtrace;
 use common_ast::Dialect;
-use common_base::base::unescape_string;
 use common_catalog::plan::DataSourceInfo;
 use common_catalog::plan::DataSourcePlan;
 use common_catalog::plan::Partitions;
@@ -611,44 +610,41 @@ pub fn parse_copy_file_format_options(
         .parse::<u64>()?;
 
     // Field delimiter.
-    let field_delimiter = unescape_string(
-        file_format_options
-            .get("field_delimiter")
-            .unwrap_or(&"".to_string()),
-    )?;
+    let field_delimiter = file_format_options
+        .get("field_delimiter")
+        .unwrap_or(&"".to_string())
+        .to_string();
 
     // Record delimiter.
-    let record_delimiter = unescape_string(
-        file_format_options
-            .get("record_delimiter")
-            .unwrap_or(&"".to_string()),
-    )?;
+    let record_delimiter = file_format_options
+        .get("record_delimiter")
+        .unwrap_or(&"".to_string())
+        .to_string();
 
     // NaN display.
-    let nan_display = unescape_string(
-        file_format_options
-            .get("nan_display")
-            .unwrap_or(&"".to_string()),
-    )?;
+    let nan_display = file_format_options
+        .get("nan_display")
+        .unwrap_or(&"".to_string())
+        .to_string();
 
     // Escape
-    let escape = unescape_string(file_format_options.get("escape").unwrap_or(&"".to_string()))?;
+    let escape = file_format_options
+        .get("escape")
+        .unwrap_or(&"".to_string())
+        .to_string();
 
     // Compression delimiter.
-    let compression = unescape_string(
-        file_format_options
-            .get("compression")
-            .unwrap_or(&"none".to_string()),
-    )?
-    .parse()
-    .map_err(ErrorCode::UnknownCompressionType)?;
+    let compression = file_format_options
+        .get("compression")
+        .unwrap_or(&"none".to_string())
+        .parse()
+        .map_err(ErrorCode::UnknownCompressionType)?;
 
     // Row tag in xml.
-    let row_tag = unescape_string(
-        file_format_options
-            .get("row_tag")
-            .unwrap_or(&"".to_string()),
-    )?;
+    let row_tag = file_format_options
+        .get("row_tag")
+        .unwrap_or(&"".to_string())
+        .to_string();
 
     Ok(FileFormatOptions {
         format: file_format,

tests/logictest/suites/base/05_ddl/05_0016_ddl_stage

@@ -23,7 +23,7 @@ statement query TTTTTITT
 desc stage test_stage_internal;
 
 ----
-test_stage_internal  Internal  StageParams { storage: Fs(StorageFsConfig { root: "_data" }) }  CopyOptions { on_error: None, size_limit: 0, split_size: 0, purge: false, single: false, max_file_size: 0 }  FileFormatOptions { format: Csv, skip_header: 0, field_delimiter: "", record_delimiter: "NONE", nan_display: "", escape: "\\", compression: Auto, row_tag: "" }  0  'root'@'127.0.0.1'
+test_stage_internal  Internal  StageParams { storage: Fs(StorageFsConfig { root: "_data" }) }  CopyOptions { on_error: None, size_limit: 0, split_size: 0, purge: false, single: false, max_file_size: 0 }  FileFormatOptions { format: Csv, skip_header: 0, field_delimiter: "", record_delimiter: "NONE", nan_display: "", escape: "\\\\", compression: Auto, row_tag: "" }  0  'root'@'127.0.0.1'
 
 statement query TTTTT
 SHOW STAGES;

tests/suites/1_stateful/05_formats/05_02_csv/05_02_01_csv_escape.sh

@@ -49,12 +49,13 @@ curl -sH "insert_sql:insert into test_csv format CSV" -H "format_escape:'\\\'" -
 echo "select * from test_csv" | $MYSQL_CLIENT_CONNECT
 echo "truncate table test_csv" | $MYSQL_CLIENT_CONNECT
 
+## just need `\\` in sql client, `\\\\` is for shell
 aws --endpoint-url http://127.0.0.1:9900/ s3 cp /tmp/escape_slash2.csv s3://testbucket/admin/data/csv/escape_slash2.csv > /dev/null 2>&1
-echo "copy into test_csv from 'fs:///tmp/escape_slash2.csv' FILE_FORMAT = (type = 'CSV' escape='\\\\\\\\')" | $MYSQL_CLIENT_CONNECT
+echo "copy into test_csv from 'fs:///tmp/escape_slash2.csv' FILE_FORMAT = (type = 'CSV' escape='\\\\')" | $MYSQL_CLIENT_CONNECT
 echo "select * from test_csv" | $MYSQL_CLIENT_CONNECT
 
 aws --endpoint-url http://127.0.0.1:9900/ s3 cp /tmp/escape_slash3.csv s3://testbucket/admin/data/csv/escape_slash3.csv > /dev/null 2>&1
-echo "copy into test_csv2 from 'fs:///tmp/escape_slash3.csv' FILE_FORMAT = (type = 'CSV' escape='\\\\\\\\' skip_header=1)" | $MYSQL_CLIENT_CONNECT
+echo "copy into test_csv2 from 'fs:///tmp/escape_slash3.csv' FILE_FORMAT = (type = 'CSV' escape='\\\\' skip_header=1)" | $MYSQL_CLIENT_CONNECT
 echo "select * from test_csv2" | $MYSQL_CLIENT_CONNECT
 
 echo "drop table if exists test_csv" | $MYSQL_CLIENT_CONNECT