Skip to content

Commit 8c9a6ff

Browse files
rvu1024rvu1024
authored
[yt provider] Allow append with same column groups (#8176)
1 parent 08f2dfb commit 8c9a6ff

File tree

17 files changed

+213
-1
lines changed

17 files changed

+213
-1
lines changed

‎ydb/library/yql/providers/yt/provider/yql_yt_datasink_type_ann.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,9 +441,10 @@ class TYtDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase {
441441
return TStatus::Error;
442442
}
443443

444-
if (initialWrite && !replaceMeta && columnGroups) {
444+
if (initialWrite && !replaceMeta && columnGroups != description.ColumnGroupSpec) {
445445
ctx.AddError(TIssue(pos, TStringBuilder()
446446
<< "Insert with "
447+
<< (outTableInfo.Epoch.GetOrElse(0) ? "different " : "")
447448
<< ToString(EYtSettingType::ColumnGroups).Quote()
448449
<< " to existing table is not allowed"));
449450
return TStatus::Error;

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4493,13 +4493,41 @@
44934493
"uri": "https://{canondata_backend}/1937027/bbc35c51807ca32a384973d8a730422ad871c54c/resource.tar.gz#test_sql2yql.test_column_group-hint_unk_col_fail_/sql.yql"
44944494
}
44954495
],
4496+
"test_sql2yql.test[column_group-insert_diff_groups1_fail]": [
4497+
{
4498+
"checksum": "e459868feddecebca5c257aa53366627",
4499+
"size": 3547,
4500+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql2yql.test_column_group-insert_diff_groups1_fail_/sql.yql"
4501+
}
4502+
],
4503+
"test_sql2yql.test[column_group-insert_diff_groups2_fail]": [
4504+
{
4505+
"checksum": "4e2ed6f8374058c2c6c2734314af8f35",
4506+
"size": 3583,
4507+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql2yql.test_column_group-insert_diff_groups2_fail_/sql.yql"
4508+
}
4509+
],
4510+
"test_sql2yql.test[column_group-insert_diff_groups3_fail]": [
4511+
{
4512+
"checksum": "9cd09aa57b21917301b4baaccd07518e",
4513+
"size": 1348,
4514+
"uri": "https://{canondata_backend}/1942415/9610360ced2221ce66696bd9f6afd9579e452c44/resource.tar.gz#test_sql2yql.test_column_group-insert_diff_groups3_fail_/sql.yql"
4515+
}
4516+
],
44964517
"test_sql2yql.test[column_group-length]": [
44974518
{
44984519
"checksum": "3bca7ac5997a4e00558b1bda7abf5e06",
44994520
"size": 4317,
45004521
"uri": "https://{canondata_backend}/1889210/67cf90fc754bd2827f5984f7117f49bafa86ef30/resource.tar.gz#test_sql2yql.test_column_group-length_/sql.yql"
45014522
}
45024523
],
4524+
"test_sql2yql.test[column_group-many_inserts]": [
4525+
{
4526+
"checksum": "c135d985e67cd616c6dfdcbc8046732a",
4527+
"size": 8009,
4528+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql2yql.test_column_group-many_inserts_/sql.yql"
4529+
}
4530+
],
45034531
"test_sql2yql.test[column_group-min_group]": [
45044532
{
45054533
"checksum": "f6bba7f288bad8466acf4544f48e1cbc",
@@ -23988,13 +24016,41 @@
2398824016
"uri": "https://{canondata_backend}/1937027/bbc35c51807ca32a384973d8a730422ad871c54c/resource.tar.gz#test_sql_format.test_column_group-hint_unk_col_fail_/formatted.sql"
2398924017
}
2399024018
],
24019+
"test_sql_format.test[column_group-insert_diff_groups1_fail]": [
24020+
{
24021+
"checksum": "f354a922ac63e9db47f95ce2f5b17187",
24022+
"size": 295,
24023+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql_format.test_column_group-insert_diff_groups1_fail_/formatted.sql"
24024+
}
24025+
],
24026+
"test_sql_format.test[column_group-insert_diff_groups2_fail]": [
24027+
{
24028+
"checksum": "995593fbae18a399f478e76bd83b364a",
24029+
"size": 303,
24030+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql_format.test_column_group-insert_diff_groups2_fail_/formatted.sql"
24031+
}
24032+
],
24033+
"test_sql_format.test[column_group-insert_diff_groups3_fail]": [
24034+
{
24035+
"checksum": "887f458985d36504bae92614369477d0",
24036+
"size": 143,
24037+
"uri": "https://{canondata_backend}/1942415/9610360ced2221ce66696bd9f6afd9579e452c44/resource.tar.gz#test_sql_format.test_column_group-insert_diff_groups3_fail_/formatted.sql"
24038+
}
24039+
],
2399124040
"test_sql_format.test[column_group-length]": [
2399224041
{
2399324042
"checksum": "cb4ec24b2d30f9c92951f9a66c928ccb",
2399424043
"size": 190,
2399524044
"uri": "https://{canondata_backend}/1889210/67cf90fc754bd2827f5984f7117f49bafa86ef30/resource.tar.gz#test_sql_format.test_column_group-length_/formatted.sql"
2399624045
}
2399724046
],
24047+
"test_sql_format.test[column_group-many_inserts]": [
24048+
{
24049+
"checksum": "8746bbe05d09032f66ca0d3aef4ed60b",
24050+
"size": 675,
24051+
"uri": "https://{canondata_backend}/1871002/424fbf9b3e2a5083fa1289e0777dbe9463d049d9/resource.tar.gz#test_sql_format.test_column_group-many_inserts_/formatted.sql"
24052+
}
24053+
],
2399824054
"test_sql_format.test[column_group-min_group]": [
2399924055
{
2400024056
"checksum": "b3461fe4fcc74722ddebbf253269167d",

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups1_fail.cfg

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
xfail
2+
in Input input.txt
3+
providers yt
4+
pragma yt.OptimizeFor="scan"

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups1_fail.sql

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
USE plato;
2+
3+
pragma yt.ColumnGroupMode="perusage";
4+
5+
$s1 = select * from Input where a != "";
6+
$s2 = select * from Input where a > "a1";
7+
8+
insert into @a
9+
select * from $s1;
10+
11+
commit;
12+
13+
insert into @a with column_groups="{a=#}"
14+
select * from $s2;

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups2_fail.cfg

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
xfail
2+
in Input input.txt
3+
out Output output.txt
4+
providers yt
5+
pragma yt.OptimizeFor="scan"

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups2_fail.sql

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
USE plato;
2+
3+
pragma yt.ColumnGroupMode="perusage";
4+
5+
$s1 = select * from Input where a != "";
6+
$s2 = select * from Input where a > "a1";
7+
8+
insert into Output
9+
select * from $s1;
10+
11+
commit;
12+
13+
insert into Output with column_groups="{a=#}"
14+
select * from $s2;

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups3_fail.cfg

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
xfail
2+
in Input input.txt
3+
out Output input.txt
4+
providers yt
5+
pragma yt.OptimizeFor="scan"

‎ydb/library/yql/tests/sql/suites/column_group/insert_diff_groups3_fail.sql

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
USE plato;
2+
3+
pragma yt.ColumnGroupMode="perusage";
4+
5+
insert into Output with column_groups="{a=#}"
6+
select * from Input where a != "";

‎ydb/library/yql/tests/sql/suites/column_group/many_inserts.cfg

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
in Input input.txt
2+
out Output output.txt
3+
providers yt
4+
pragma yt.OptimizeFor="scan"

‎ydb/library/yql/tests/sql/suites/column_group/many_inserts.sql

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
USE plato;
2+
3+
pragma yt.ColumnGroupMode="perusage";
4+
5+
$s1 = select * from Input where a != "";
6+
$s2 = select * from Input where a > "a1";
7+
8+
insert into @a with column_groups="{a=#}"
9+
select * from $s1;
10+
11+
insert into @b
12+
select * from $s1;
13+
14+
insert into @c
15+
select * from $s1;
16+
17+
insert into Output with column_groups="{a=#}"
18+
select * from $s1;
19+
20+
commit;
21+
22+
insert into @a with column_groups="{a=#}"
23+
select * from $s2;
24+
25+
insert into @b
26+
select * from $s2;
27+
28+
insert into @c with column_groups="{default=#}"
29+
select * from $s2;
30+
31+
insert into Output with column_groups="{a=#}"
32+
select * from $s2;

‎ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -777,6 +777,32 @@
777777
"uri": "https://{canondata_backend}/1936947/e09226cb0878d9882f65262a57b6b08cedb2f071/resource.tar.gz#test.test_column_group-hint-disable-Results_/Output3.yqlrun.txt.attr"
778778
}
779779
],
780+
"test.test[column_group-many_inserts--Debug]": [
781+
{
782+
"checksum": "3b938a7ef06bfdca107284471470ca03",
783+
"size": 3822,
784+
"uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Debug_/opt.yql"
785+
}
786+
],
787+
"test.test[column_group-many_inserts--Plan]": [
788+
{
789+
"checksum": "e61f23007f4d36c710b7255a1e03733e",
790+
"size": 30291,
791+
"uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Plan_/plan.txt"
792+
}
793+
],
794+
"test.test[column_group-many_inserts--Results]": [
795+
{
796+
"checksum": "94fde3f913511b2af79a70f97849f2c3",
797+
"size": 300,
798+
"uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Results_/Output.txt"
799+
},
800+
{
801+
"checksum": "685987a135166ca30afe0a22810c35c3",
802+
"size": 1078,
803+
"uri": "https://{canondata_backend}/1903885/d42df107858b9345ba042b91c20d00a7ded84888/resource.tar.gz#test.test_column_group-many_inserts--Results_/Output.yqlrun.txt.attr"
804+
}
805+
],
780806
"test.test[column_order-winfunc-default.txt-Debug]": [
781807
{
782808
"checksum": "4900c9724073dd97882884caffe527ad",

‎ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,13 @@
845845
"uri": "https://{canondata_backend}/1871102/7ae49534755abc3a8ee593205c473cd1d8570946/resource.tar.gz#test.test_blocks-struct_type--Results_/results.txt"
846846
}
847847
],
848+
"test.test[column_group-insert_diff_groups2_fail--Debug]": [],
849+
"test.test[column_group-insert_diff_groups2_fail--Plan]": [],
850+
"test.test[column_group-insert_diff_groups2_fail--Results]": [
851+
{
852+
"uri": "file://test.test_column_group-insert_diff_groups2_fail--Results_/extracted"
853+
}
854+
],
848855
"test.test[column_group-min_group-default.txt-Debug]": [
849856
{
850857
"checksum": "1aaf16e9c5132c0572386fd4d56c7277",

‎ydb/library/yql/tests/sql/yt_native_file/part14/canondata/test.test_column_group-insert_diff_groups2_fail--Results_/extracted

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<tmp_path>/program.sql:<main>: Error: Type annotation
2+
3+
<tmp_path>/program.sql:<main>:14:39: Error: At function: YtWriteTable!
4+
insert into Output with column_groups="{a=#}"
5+
^
6+
<tmp_path>/program.sql:<main>:14:39: Error: Insert with different "column_groups" to existing table is not allowed
7+
insert into Output with column_groups="{a=#}"
8+
^

‎ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -739,6 +739,13 @@
739739
"uri": "https://{canondata_backend}/1899731/7897b30eef0bb27e4ec2a81827074c48697423a5/resource.tar.gz#test.test_column_group-groups-lookup-Results_/results.txt"
740740
}
741741
],
742+
"test.test[column_group-insert_diff_groups1_fail--Debug]": [],
743+
"test.test[column_group-insert_diff_groups1_fail--Plan]": [],
744+
"test.test[column_group-insert_diff_groups1_fail--Results]": [
745+
{
746+
"uri": "file://test.test_column_group-insert_diff_groups1_fail--Results_/extracted"
747+
}
748+
],
742749
"test.test[column_order-insert_with_desc_sort_and_native_types-default.txt-Debug]": [
743750
{
744751
"checksum": "8d9d1101d61c96254d2d67b4736ed680",

‎ydb/library/yql/tests/sql/yt_native_file/part15/canondata/test.test_column_group-insert_diff_groups1_fail--Results_/extracted

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<tmp_path>/program.sql:<main>: Error: Type annotation
2+
3+
<tmp_path>/program.sql:<main>:14:35: Error: At function: YtWriteTable!
4+
insert into @a with column_groups="{a=#}"
5+
^
6+
<tmp_path>/program.sql:<main>:14:35: Error: Insert with different "column_groups" to existing table is not allowed
7+
insert into @a with column_groups="{a=#}"
8+
^

‎ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -623,6 +623,13 @@
623623
"uri": "file://test.test_column_group-hint_short_grp_fail--Results_/extracted"
624624
}
625625
],
626+
"test.test[column_group-insert_diff_groups3_fail--Debug]": [],
627+
"test.test[column_group-insert_diff_groups3_fail--Plan]": [],
628+
"test.test[column_group-insert_diff_groups3_fail--Results]": [
629+
{
630+
"uri": "file://test.test_column_group-insert_diff_groups3_fail--Results_/extracted"
631+
}
632+
],
626633
"test.test[column_order-ordered_plus_native--Debug]": [
627634
{
628635
"checksum": "8ce624d459aee467421b612822e10b6a",

‎ydb/library/yql/tests/sql/yt_native_file/part9/canondata/test.test_column_group-insert_diff_groups3_fail--Results_/extracted

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<tmp_path>/program.sql:<main>: Error: Type annotation
2+
3+
<tmp_path>/program.sql:<main>:6:39: Error: At function: YtWriteTable!
4+
insert into Output with column_groups="{a=#}"
5+
^
6+
<tmp_path>/program.sql:<main>:6:39: Error: Insert with "column_groups" to existing table is not allowed
7+
insert into Output with column_groups="{a=#}"
8+
^

0 commit comments

Comments
 (0)