Skip to content

Commit 55eedf7

Browse files
mxkovalevmxkovalev
authored
YQL: Fuse reduces with presort settings (#8117)
1 parent 6eb6b83 commit 55eedf7

File tree

9 files changed

+174
-2
lines changed

9 files changed

+174
-2
lines changed

‎ydb/library/yql/providers/yt/provider/phy_opt/yql_yt_phy_opt_fuse.cpp

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,16 +120,23 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no
120120
!NYql::HasSetting(innerReduce.Settings().Ref(), EYtSettingType::ReduceBy)) {
121121
return node;
122122
}
123-
124123
if (NYql::HasSetting(outerReduce.Settings().Ref(), EYtSettingType::SortBy)) {
125-
return node;
124+
auto innerSortBy = NYql::GetSettingAsColumnList(innerReduce.Settings().Ref(), EYtSettingType::SortBy);
125+
auto outerSortBy = NYql::GetSettingAsColumnList(outerReduce.Settings().Ref(), EYtSettingType::SortBy);
126+
if (outerSortBy.size() > innerSortBy.size()) {
127+
return node;
128+
}
129+
if (!std::equal(outerSortBy.cbegin(), outerSortBy.cend(), innerSortBy.cbegin())) {
130+
return node;
131+
}
126132
}
127133

128134
if (NYql::HasSettingsExcept(innerReduce.Settings().Ref(), EYtSettingType::ReduceBy |
129135
EYtSettingType::KeySwitch |
130136
EYtSettingType::Flow |
131137
EYtSettingType::FirstAsPrimary |
132138
EYtSettingType::SortBy |
139+
EYtSettingType::KeepSorted |
133140
EYtSettingType::NoDq)) {
134141
return node;
135142
}
@@ -227,6 +234,11 @@ TMaybeNode<TExprBase> TYtPhysicalOptProposalTransformer::FuseReduce(TExprBase no
227234
newSettings = NYql::AddSetting(*newSettings, EYtSettingType::NoDq, {}, ctx);
228235
}
229236

237+
if (NYql::HasSetting(outerReduce.Settings().Ref(), EYtSettingType::KeepSorted) &&
238+
!NYql::HasSetting(innerReduce.Settings().Ref(), EYtSettingType::KeepSorted)) {
239+
newSettings = NYql::AddSetting(*newSettings, EYtSettingType::KeepSorted, {}, ctx);
240+
}
241+
230242
return Build<TYtReduce>(ctx, node.Pos())
231243
.InitFrom(outerReduce)
232244
.World<TCoSync>()

‎ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2410,6 +2410,28 @@
24102410
}
24112411
],
24122412
"test.test[pragma-yson_auto_convert--Results]": [],
2413+
"test.test[produce-fuse_reduces_with_presort--Analyze]": [
2414+
{
2415+
"checksum": "325a59b196f61e264182ef061c4cc672",
2416+
"size": 13222,
2417+
"uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Analyze_/plan.txt"
2418+
}
2419+
],
2420+
"test.test[produce-fuse_reduces_with_presort--Debug]": [
2421+
{
2422+
"checksum": "161aa9d1d7f46024786b2745eefd0bfc",
2423+
"size": 4976,
2424+
"uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql_patched"
2425+
}
2426+
],
2427+
"test.test[produce-fuse_reduces_with_presort--Plan]": [
2428+
{
2429+
"checksum": "325a59b196f61e264182ef061c4cc672",
2430+
"size": 13222,
2431+
"uri": "https://{canondata_backend}/1937367/c8f509a79779b30b722211cfeb063fe74251b5ea/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt"
2432+
}
2433+
],
2434+
"test.test[produce-fuse_reduces_with_presort--Results]": [],
24132435
"test.test[produce-yql-10297-default.txt-Analyze]": [
24142436
{
24152437
"checksum": "a3b64a2cf9903b3868a2dd88a18fc46e",

‎ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2589,6 +2589,20 @@
25892589
"uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg_catalog-pg_timezone_abbrevs-default.txt-Plan_/plan.txt"
25902590
}
25912591
],
2592+
"test.test[produce-fuse_reduces_with_presort--Debug]": [
2593+
{
2594+
"checksum": "a0be01e2962b39871f4097f1d598a1ca",
2595+
"size": 7199,
2596+
"uri": "https://{canondata_backend}/1942525/70eec467682e96dfaac0f3c809005b8a22cce54f/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql_patched"
2597+
}
2598+
],
2599+
"test.test[produce-fuse_reduces_with_presort--Plan]": [
2600+
{
2601+
"checksum": "b7d4ddc5c9f07fedb74a72b0789bd050",
2602+
"size": 7633,
2603+
"uri": "https://{canondata_backend}/1942525/70eec467682e96dfaac0f3c809005b8a22cce54f/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt"
2604+
}
2605+
],
25922606
"test.test[produce-reduce_lambda_list_mem-default.txt-Debug]": [
25932607
{
25942608
"checksum": "940dd13b9a62a90b5c95a4a12a765fe7",

‎ydb/library/yql/tests/sql/sql2yql/canondata/result.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15336,6 +15336,13 @@
1533615336
"uri": "https://{canondata_backend}/1784117/d56ae82ad9d30397a41490647be1bd2124718f98/resource.tar.gz#test_sql2yql.test_produce-discard_reduce_lambda_/sql.yql"
1533715337
}
1533815338
],
15339+
"test_sql2yql.test[produce-fuse_reduces_with_presort]": [
15340+
{
15341+
"checksum": "e379da7ed6a051092b9b7125ec129f09",
15342+
"size": 11861,
15343+
"uri": "https://{canondata_backend}/1923547/a000f225caa0fe7476a3e0c0740662cd5bbe1d5e/resource.tar.gz#test_sql2yql.test_produce-fuse_reduces_with_presort_/sql.yql"
15344+
}
15345+
],
1533915346
"test_sql2yql.test[produce-native_desc_reduce_with_presort]": [
1534015347
{
1534115348
"checksum": "78eedc889a54e4499e62fb3752015a4a",
@@ -31688,6 +31695,13 @@
3168831695
"uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_produce-discard_reduce_lambda_/formatted.sql"
3168931696
}
3169031697
],
31698+
"test_sql_format.test[produce-fuse_reduces_with_presort]": [
31699+
{
31700+
"checksum": "8f8335d2d11bff52392da6cc36645c55",
31701+
"size": 1636,
31702+
"uri": "https://{canondata_backend}/1923547/a000f225caa0fe7476a3e0c0740662cd5bbe1d5e/resource.tar.gz#test_sql_format.test_produce-fuse_reduces_with_presort_/formatted.sql"
31703+
}
31704+
],
3169131705
"test_sql_format.test[produce-native_desc_reduce_with_presort]": [
3169231706
{
3169331707
"checksum": "6af6ad9ee36a41b35cec0e63c725eae8",

‎ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
in Input input_sorted.txt

‎ydb/library/yql/tests/sql/suites/produce/fuse_reduces_with_presort.sql

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
USE plato;
2+
3+
$reduce = ($_, $TableRows) -> {
4+
return Yql::Condense1(
5+
$TableRows,
6+
($item) -> ($item),
7+
($_, $_) -> (false),
8+
($item, $_) -> ($item)
9+
);
10+
};
11+
12+
$stream =
13+
select * from Input;
14+
15+
--
16+
17+
$stream1 = Reduce $stream
18+
presort value1
19+
on key, subkey
20+
using $reduce(TableRow())
21+
assume order by key, subkey, value1;
22+
23+
$stream1 = Reduce $stream1
24+
presort value1
25+
on key, subkey
26+
using $reduce(TableRow())
27+
assume order by key, subkey, value1;
28+
29+
--
30+
31+
$stream2 = Reduce $stream
32+
presort value1, value2
33+
on key, subkey
34+
using $reduce(TableRow())
35+
assume order by key, subkey, value1, value2;
36+
37+
$stream2 = Reduce $stream2
38+
presort value1
39+
on key, subkey
40+
using $reduce(TableRow())
41+
assume order by key, subkey, value1;
42+
43+
--
44+
45+
$stream3 = Reduce $stream
46+
presort value1, value2, value3
47+
on key, subkey
48+
using $reduce(TableRow())
49+
assume order by key, subkey, value1, value2, value3;
50+
51+
$stream3 = Reduce $stream3
52+
on key, subkey
53+
using $reduce(TableRow())
54+
assume order by key, subkey;
55+
56+
select
57+
*
58+
from $stream1
59+
ASSUME ORDER BY `key`, `subkey`;
60+
61+
select
62+
*
63+
from $stream2
64+
ASSUME ORDER BY `key`, `subkey`;
65+
66+
select
67+
*
68+
from $stream3
69+
ASSUME ORDER BY `key`, `subkey`;

‎ydb/library/yql/tests/sql/suites/produce/input_sorted.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"key"="020";"subkey"="1";"value1"="abc";"value2"="aabbcc";"value3"="aa"};
2+
{"key"="075";"subkey"="2";"value1"="ddd";"value2"="dddddd";"value3"="bb"};
3+
{"key"="150";"subkey"="3";"value1"="q";"value2"="qq";"value3"="cc"};
4+
{"key"="800";"subkey"="4";"value1"="qzz";"value2"="qqzzzz";"value3"="dd"};

‎ydb/library/yql/tests/sql/suites/produce/input_sorted.txt.attr

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"_yql_row_spec"={
3+
"Type"=["StructType";[
4+
["key";["DataType";"String"]];
5+
["subkey";["DataType";"String"]];
6+
["value1";["DataType";"String"]];
7+
["value2";["DataType";"String"]];
8+
["value3";["DataType";"String"]]
9+
]];
10+
"SortDirections"=[1;1;1;1;1];
11+
"SortedBy"=["key";"subkey";"value1";"value2";"value3"];
12+
"SortedByTypes"=[["DataType";"String"];["DataType";"String"];["DataType";"String"];["DataType";"String"];["DataType";"String"]];
13+
"SortMembers"=["key";"subkey";"value1";"value2";"value3"];
14+
}
15+
}

‎ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2261,6 +2261,27 @@
22612261
"uri": "https://{canondata_backend}/1903280/4c77300cd3fef018d27d7f75b6ff956e63258b21/resource.tar.gz#test.test_pragma-yson_auto_convert--Results_/results.txt"
22622262
}
22632263
],
2264+
"test.test[produce-fuse_reduces_with_presort--Debug]": [
2265+
{
2266+
"checksum": "7eec75c62c340d46b9fd0d652345e21a",
2267+
"size": 3577,
2268+
"uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Debug_/opt.yql"
2269+
}
2270+
],
2271+
"test.test[produce-fuse_reduces_with_presort--Plan]": [
2272+
{
2273+
"checksum": "9e8381980dc1e802f465fa49ee5dfd14",
2274+
"size": 11259,
2275+
"uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Plan_/plan.txt"
2276+
}
2277+
],
2278+
"test.test[produce-fuse_reduces_with_presort--Results]": [
2279+
{
2280+
"checksum": "487e3e6ee23dd6103b213486901febc4",
2281+
"size": 7301,
2282+
"uri": "https://{canondata_backend}/1889210/f2ee730cb1f26eb197a3bfb0493ec60757c5a916/resource.tar.gz#test.test_produce-fuse_reduces_with_presort--Results_/results.txt"
2283+
}
2284+
],
22642285
"test.test[produce-yql-10297-default.txt-Debug]": [
22652286
{
22662287
"checksum": "6ee64112721e725a4bd2da387df401e6",

0 commit comments

Comments
 (0)