update streaming load docs and tests

Author: ariesdevil

docs/doc/12-load-data/02-local.md

@@ -43,7 +43,7 @@ CREATE TABLE books
 Create and send the API request with the following scripts:
 
 ```bash
-curl -XPUT 'http://root:@127.0.0.1:8000/v1/streaming_load' -H 'insert_sql: insert into book_db.books format CSV' -H 'format_skip_header: 0' -H 'format_field_delimiter: ,' -H 'format_record_delimiter: \n' -F 'upload=@"./books.csv"'
+curl -XPUT 'http://root:@127.0.0.1:8000/v1/streaming_load' -H 'insert_sql: insert into book_db.books file_format = (type = "CSV" skip_header = 0 field_delimiter = "," record_delimiter = "\n")' -F 'upload=@"./books.csv"'
 ```
 
 Response Example:
@@ -101,7 +101,7 @@ CREATE TABLE bookcomments
 Create and send the API request with the following scripts:
 
 ```bash
-curl -XPUT 'http://root:@127.0.0.1:8000/v1/streaming_load' -H 'insert_sql: insert into book_db.bookcomments(title,author,date)format CSV' -H 'format_skip_header: 0' -H 'format_field_delimiter: ,' -H 'format_record_delimiter: \n' -F 'upload=@"./books.csv"'
+curl -XPUT 'http://root:@127.0.0.1:8000/v1/streaming_load' -H 'insert_sql: insert into book_db.bookcomments(title,author,date) file_format = (type = "CSV" skip_header = 0 field_delimiter = "," record_delimiter = "\n")' -F 'upload=@"./books.csv"'
 ```
 
 Notice that the `insert_sql` part above specifies the columns (title, author, and date) to match the loaded data.

docs/doc/21-use-cases/01-analyze-ontime-with-databend-on-ec2-and-s3.md

@@ -48,7 +48,7 @@ unzip t_ontime.csv.zip
 ```
 
 ```shell title='Load TSV files into Databend'
-curl -H "insert_sql:insert into ontime format TSV" -H "format_skip_header:0" -F "upload=@t_ontime.csv"  -XPUT http://root:@127.0.0.1:8000/v1/streaming_load
+curl -H "insert_sql:insert into ontime file_format = (type = 'TSV' skip_header = 0)" -F "upload=@t_ontime.csv"  -XPUT http://root:@127.0.0.1:8000/v1/streaming_load
 ```
 
 :::tip

docs/doc/21-use-cases/05-analyze-hits-dataset-with-databend.md

@@ -47,7 +47,7 @@ gzip -d hits_1m.csv.gz
 ```
 
 ```shell title='Load CSV files into Databend'
-curl -H "insert_sql:insert into hits format TSV"  -F  "upload=@./hits_1m.tsv"  -XPUT http://user1:abc123@127.0.0.1:8000/v1/streaming_load
+curl -H "insert_sql:insert into hits file_format = (type = 'TSV')"  -F  "upload=@./hits_1m.tsv"  -XPUT http://user1:abc123@127.0.0.1:8000/v1/streaming_load
 ```
 
 ## Step 3. Queries

tests/suites/0_stateless/13_tpch/13_0000_prepare.sh

@@ -111,5 +111,6 @@ tar -zxf ${CURDIR}/data/tpch.tar.gz -C ${CURDIR}/data
 for t in customer lineitem nation orders partsupp part region supplier
 do
     echo "$t"
-    curl -s -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" -H 'insert_sql: insert into '$t' format CSV' -H 'format_skip_header: 0' -H 'format_field_delimiter:|' -H 'format_record_delimiter: \n' -F 'upload=@"'${CURDIR}'/data/tests/suites/0_stateless/13_tpch/data/'$t'.tbl"' > /dev/null 2>&1
+    insert_sql="insert into $t file_format = (type = 'CSV' skip_header = 0 field_delimiter = '|' record_delimiter = '\n')"
+    curl -s -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" -H "insert_sql: ${insert_sql}" -F 'upload=@"'${CURDIR}'/data/tests/suites/0_stateless/13_tpch/data/'$t'.tbl"' > /dev/null 2>&1
 done

tests/suites/1_stateful/01_load_v2/01_0000_streaming_load_books.sh

@@ -14,11 +14,11 @@ echo "CREATE TABLE books
 );" | $MYSQL_CLIENT_CONNECT
 
 # load csv
-curl -H "insert_sql:insert into books format CSV" -F "upload=@${CURDIR}/books.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
+curl -H "insert_sql:insert into books file_format = (type = 'CSV')" -F "upload=@${CURDIR}/books.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 echo "select count(), count_if(title is null), count_if(author is null), count_if(date is null), count_if(publish_time is null) from books " |  $MYSQL_CLIENT_CONNECT
 
 # load tsv
-curl -H "insert_sql:insert into books format TSV" -F "upload=@${CURDIR}/books.tsv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
+curl -H "insert_sql:insert into books file_format = (type = 'TSV')" -F "upload=@${CURDIR}/books.tsv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 echo "select count(), count_if(title is null), count_if(author is null), count_if(date is null), count_if(publish_time is null) from books " |  $MYSQL_CLIENT_CONNECT
 
 

tests/suites/1_stateful/01_load_v2/01_0004_streaming_parquet_int96.sh

@@ -19,7 +19,7 @@ if [ $? -ne 0 ]; then
 fi
 
 # load parquet
-curl -H "insert_sql:insert into mytime format Parquet" -F "upload=@/tmp/mytime.parquet" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
+curl -H "insert_sql:insert into mytime file_format = (type = 'Parquet')" -F "upload=@/tmp/mytime.parquet" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 echo "select * from mytime" | $MYSQL_CLIENT_CONNECT
 echo "drop table mytime;" | $MYSQL_CLIENT_CONNECT
 

tests/suites/1_stateful/01_load_v2/01_0004_streaming_variant_load.sh

@@ -32,12 +32,13 @@ if [ $? -ne 0 ]; then
 fi
 
 # load csv
+# todo(ariesdevil): change to new syntax when format_quote landing
 curl -H "insert_sql:insert into variant_test format Csv" -H "format_skip_header:0" -H 'format_field_delimiter: ,' -H 'format_record_delimiter: \n' -H "format_quote: \'" -F "upload=@/tmp/json_sample1.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 curl -H "insert_sql:insert into variant_test format Csv" -H "format_skip_header:0" -H 'format_field_delimiter: |' -H 'format_record_delimiter: \n' -H "format_quote: \'" -F "upload=@/tmp/json_sample2.csv" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 echo "select * from variant_test order by Id asc;" | $MYSQL_CLIENT_CONNECT
 
 # load ndjson
-curl -H "insert_sql:insert into variant_test2 format NdJson" -H "format_skip_header:0" -F "upload=@/tmp/json_sample.ndjson" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
+curl -H "insert_sql:insert into variant_test2 file_format = (type = 'NdJson' skip_header = 0)" -F "upload=@/tmp/json_sample.ndjson" -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" > /dev/null 2>&1
 echo "select * from variant_test2 order by b asc;" | $MYSQL_CLIENT_CONNECT
 
 echo "drop table variant_test;" | $MYSQL_CLIENT_CONNECT

tests/suites/1_stateful/05_formats/05_00_load_unload_all.sh

@@ -35,7 +35,7 @@ test_format() {
 
 	echo "truncate table test_load_unload" | $MYSQL_CLIENT_CONNECT
 
-	curl -sH "insert_sql:insert into test_load_unload format ${1}" \
+	curl -sH "insert_sql:insert into test_load_unload file_format = (type = '${1}')" \
 	-F "upload=@/tmp/test_load_unload.txt" \
 	-u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"
 

tests/suites/1_stateful/05_formats/05_01_compact/05_01_01_load_compact_streaming_load.sh

@@ -17,7 +17,7 @@ echo "CREATE TABLE t1
 );" | $MYSQL_CLIENT_CONNECT
 
 echo "---load"
-curl -sH "insert_sql:insert into t1 format csv" \
+curl -sH "insert_sql:insert into t1 file_format = (type = 'CSV')" \
 -F "upload=@${DATA}" \
 -H "input_read_buffer_size: 100" \
 -u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"

tests/suites/1_stateful/05_formats/05_01_load_unload_simple.sh

@@ -31,7 +31,7 @@ test_format() {
 
 	echo "truncate table test_load_unload" | $MYSQL_CLIENT_CONNECT
 
-	curl -sH "insert_sql:insert into test_load_unload format ${1}" \
+	curl -sH "insert_sql:insert into test_load_unload file_format = (type = '${1}')" \
 	-F "upload=@/tmp/test_load_unload.txt" \
 	-u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"