fix: max_files not work for transform on ndjson/csv/tsv. (#15441)

Author: youngsofun

src/query/sql/src/planner/binder/table.rs

@@ -942,7 +942,7 @@ impl Binder {
                     schema,
                     stage_info,
                     files_info,
-                    files_to_copy: None,
+                    files_to_copy,
                     duplicated_files_detected: vec![],
                     is_select: true,
                     default_values: None,
@@ -977,7 +977,7 @@ impl Binder {
                     schema,
                     stage_info,
                     files_info,
-                    files_to_copy: None,
+                    files_to_copy,
                     duplicated_files_detected: vec![],
                     is_select: true,
                     default_values: None,

tests/suites/1_stateful/00_stage/00_0004_copy_with_max_files.result

@@ -1,16 +1,32 @@
---- force = false, purge = false
+--- force = false, purge = false, transform = false
 copied 2 files with 4 rows, remain 3 files
 copied 1 files with 6 rows, remain 3 files
 copied 0 files with 6 rows, remain 3 files
---- force = false, purge = true
+--- force = false, purge = true, transform = false
 copied 2 files with 4 rows, remain 1 files
 copied 1 files with 6 rows, remain 0 files
 copied 0 files with 6 rows, remain 0 files
---- force = true, purge = false
+--- force = true, purge = false, transform = false
 copied 2 files with 4 rows, remain 3 files
 copied 2 files with 8 rows, remain 3 files
 copied 2 files with 12 rows, remain 3 files
---- force = true, purge = true
+--- force = true, purge = true, transform = false
+copied 2 files with 4 rows, remain 1 files
+copied 1 files with 6 rows, remain 0 files
+copied 0 files with 6 rows, remain 0 files
+--- force = false, purge = false, transform = true
+copied 2 files with 4 rows, remain 3 files
+copied 1 files with 6 rows, remain 3 files
+copied 0 files with 6 rows, remain 3 files
+--- force = false, purge = true, transform = true
+copied 2 files with 4 rows, remain 1 files
+copied 1 files with 6 rows, remain 0 files
+copied 0 files with 6 rows, remain 0 files
+--- force = true, purge = false, transform = true
+copied 2 files with 4 rows, remain 3 files
+copied 2 files with 8 rows, remain 3 files
+copied 2 files with 12 rows, remain 3 files
+--- force = true, purge = true, transform = true
 copied 2 files with 4 rows, remain 1 files
 copied 1 files with 6 rows, remain 0 files
 copied 0 files with 6 rows, remain 0 files

tests/suites/1_stateful/00_stage/00_0004_copy_with_max_files.sh

@@ -6,16 +6,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # Should be <root>/tests/data/
 
-for force in 'false'  'true'
+for transform in 'false'  'true'
 do
-	for purge in 'false'  'true'
+	for force in 'false'  'true'
 	do
-		table="test_max_files_force_${force}_purge_${purge}"
-		echo "drop table if exists ${table}" | $BENDSQL_CLIENT_CONNECT
-		echo "CREATE TABLE ${table} (
-      id INT,
-      c1 INT
-    ) ENGINE=FUSE;" | $BENDSQL_CLIENT_CONNECT
+		for purge in 'false'  'true'
+		do
+			table="test_max_files_force_${force}_purge_${purge}_transform_${transform}"
+			echo "drop table if exists ${table}" | $BENDSQL_CLIENT_CONNECT
+			echo "CREATE TABLE ${table} (
+				id INT,
+				c1 INT
+			) ENGINE=FUSE;" | $BENDSQL_CLIENT_CONNECT
+		done
 	done
 done
 
@@ -39,28 +42,39 @@ cat << EOF > /tmp/00_0004/f3.csv
 EOF
 }
 
-for force in 'false'  'true'
+for transform in 'false'  'true'
 do
-	for purge in 'false'  'true'
+	for force in 'false'  'true'
 	do
-		gen_files
-		echo "--- force = ${force}, purge = ${purge}"
-		for i in {1..3}
+		for purge in 'false'  'true'
 		do
-			table="test_max_files_force_${force}_purge_${purge}"
-			copied=$(echo "copy into ${table} from 'fs:///tmp/00_0004/' FILE_FORMAT = (type = CSV) max_files=2 force=${force} purge=${purge}" | $BENDSQL_CLIENT_CONNECT | wc -l |  sed 's/ //g')
-			copied_rows=$(echo "select count(*) from ${table}" | $BENDSQL_CLIENT_CONNECT)
-		  remain=$(ls -1 /tmp/00_0004/ | wc -l |  sed 's/ //g')
-			echo "copied ${copied} files with ${copied_rows} rows, remain ${remain} files"
+			gen_files
+			echo "--- force = ${force}, purge = ${purge}, transform = ${transform}"
+			for i in {1..3}
+			do
+				table="test_max_files_force_${force}_purge_${purge}_transform_${transform}"
+				if [ "$transform" = true ]; then
+					copied=$(echo "copy into ${table} from 'fs:///tmp/00_0004/' FILE_FORMAT = (type = CSV) max_files=2 force=${force} purge=${purge}" | $BENDSQL_CLIENT_CONNECT | wc -l |  sed 's/ //g')
+        else
+					copied=$(echo "copy into ${table} from (select \$1, \$2 from 'fs:///tmp/00_0004/') FILE_FORMAT = (type = CSV) max_files=2 force=${force} purge=${purge}" | $BENDSQL_CLIENT_CONNECT | wc -l |  sed 's/ //g')
+        fi
+				copied_rows=$(echo "select count(*) from ${table}" | $BENDSQL_CLIENT_CONNECT)
+				remain=$(ls -1 /tmp/00_0004/ | wc -l |  sed 's/ //g')
+				echo "copied ${copied} files with ${copied_rows} rows, remain ${remain} files"
+			done
 		done
 	done
 done
 
-for force in 'false'  'true'
+for transform in 'false'  'true'
 do
-	for purge in 'false'  'true'
+	for force in 'false'  'true'
 	do
-		echo "drop table if exists test_max_files_force_${force}_purge_${purge}" | $BENDSQL_CLIENT_CONNECT
+		for purge in 'false'  'true'
+		do
+			table="test_max_files_force_${force}_purge_${purge}"
+			echo "drop table if exists ${table}" | $BENDSQL_CLIENT_CONNECT
+		done
 	done
 done