Skip to content

Commit a5cc14d

Browse files
nepalnepal
authored
Relax FlatMap over Extend requirements (#8626)
1 parent e1da3b1 commit a5cc14d

File tree

28 files changed

+136
-136
lines changed

28 files changed

+136
-136
lines changed

‎ydb/library/yql/core/common_opt/yql_co_flow1.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1381,7 +1381,7 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx,
13811381
}
13821382
}
13831383

1384-
if (node->Head().IsCallable(Ordered ? "OrderedExtend" : "Extend") &&
1384+
if (node->Head().IsCallable({"OrderedExtend", "Extend", "Merge"}) &&
13851385
// constraints below can not be derived for (Ordered)Extend
13861386
!node->GetConstraint<TSortedConstraintNode>() &&
13871387
!node->GetConstraint<TPartOfSortedConstraintNode>() &&
@@ -1391,9 +1391,9 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx,
13911391
!node->GetConstraint<TPartOfDistinctConstraintNode>())
13921392
{
13931393
auto canPush = [&](const auto& child) {
1394-
// we push FlatMap over Extend only if it can later be fused with child
1395-
return child->IsCallable({Ordered ? "OrderedFlatMap" : "FlatMap", "GroupByKey", "CombineByKey", "PartitionByKey", "PartitionsByKeys", "ShuffleByKeys",
1396-
"ListIf", "FlatListIf", "AsList", "ToList"}) && optCtx.IsSingleUsage(*child);
1394+
// we push FlatMap over Extend only if it can later be fused with or pushed down through child
1395+
return child->IsCallable({"OrderedFlatMap", "FlatMap", "GroupByKey", "CombineByKey", "PartitionByKey", "PartitionsByKeys", "ShuffleByKeys",
1396+
"Aggregate", "EquiJoin", "ListIf", "FlatListIf", "AsList", "ToList"});
13971397
};
13981398
if (AllOf(node->Head().ChildrenList(), canPush)) {
13991399
TExprNodeList newChildren;

‎ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1952,9 +1952,9 @@
19521952
],
19531953
"test.test[pg-select_agg_gs_rollup-default.txt-Debug]": [
19541954
{
1955-
"checksum": "4e890852d5e654d71472abbac4a10c5f",
1956-
"size": 1166,
1957-
"uri": "https://{canondata_backend}/1923547/320f607d9e9c19a93a835d3183938f1fba6dd52c/resource.tar.gz#test.test_pg-select_agg_gs_rollup-default.txt-Debug_/opt.yql_patched"
1955+
"checksum": "652847daf547f66e473bb2ee4760768b",
1956+
"size": 1132,
1957+
"uri": "https://{canondata_backend}/1925821/cfff423c8d7238e8abebca9535bd33e932257ec1/resource.tar.gz#test.test_pg-select_agg_gs_rollup-default.txt-Debug_/opt.yql_patched"
19581958
}
19591959
],
19601960
"test.test[pg-select_agg_gs_rollup-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2552,9 +2552,9 @@
25522552
],
25532553
"test.test[pg-select_join_left_equi_and_one-default.txt-Debug]": [
25542554
{
2555-
"checksum": "9b4e51d0d9877fa658883ecd22f6b177",
2556-
"size": 2262,
2557-
"uri": "https://{canondata_backend}/1031349/e4ef3d587c2530bf47672a5783f4b0d0a4b560fa/resource.tar.gz#test.test_pg-select_join_left_equi_and_one-default.txt-Debug_/opt.yql_patched"
2555+
"checksum": "77cb4b10c107e00ae3e34e647b4ead26",
2556+
"size": 1794,
2557+
"uri": "https://{canondata_backend}/1900335/94df111ee1e3cf59de6d62f855e8e85690405b51/resource.tar.gz#test.test_pg-select_join_left_equi_and_one-default.txt-Debug_/opt.yql_patched"
25582558
}
25592559
],
25602560
"test.test[pg-select_join_left_equi_and_one-default.txt-Plan]": [
@@ -2574,9 +2574,9 @@
25742574
],
25752575
"test.test[pg-select_join_right_equi_and_one-default.txt-Debug]": [
25762576
{
2577-
"checksum": "97db91c776c211ba5e85c094d42ee0cb",
2578-
"size": 2264,
2579-
"uri": "https://{canondata_backend}/1031349/e4ef3d587c2530bf47672a5783f4b0d0a4b560fa/resource.tar.gz#test.test_pg-select_join_right_equi_and_one-default.txt-Debug_/opt.yql_patched"
2577+
"checksum": "edebe2a5b6572ec89cb74404a8bb94f9",
2578+
"size": 1796,
2579+
"uri": "https://{canondata_backend}/1900335/94df111ee1e3cf59de6d62f855e8e85690405b51/resource.tar.gz#test.test_pg-select_join_right_equi_and_one-default.txt-Debug_/opt.yql_patched"
25802580
}
25812581
],
25822582
"test.test[pg-select_join_right_equi_and_one-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,9 +284,9 @@
284284
],
285285
"test.test[aggregate-group_by_cube_expr_trio--Debug]": [
286286
{
287-
"checksum": "17691e421b8446c9f5e8d1e1e3367214",
288-
"size": 7033,
289-
"uri": "https://{canondata_backend}/1847551/0c915a458dc980cf235b6b87ce642ec9efee9e93/resource.tar.gz#test.test_aggregate-group_by_cube_expr_trio--Debug_/opt.yql_patched"
287+
"checksum": "478bdde7a652428a5eedeae7427accba",
288+
"size": 7036,
289+
"uri": "https://{canondata_backend}/1942100/ea5d6f04ae9c974212a40f8582f093e394dd64d6/resource.tar.gz#test.test_aggregate-group_by_cube_expr_trio--Debug_/opt.yql_patched"
290290
}
291291
],
292292
"test.test[aggregate-group_by_cube_expr_trio--Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2094,9 +2094,9 @@
20942094
],
20952095
"test.test[pg-select_agg_gs_grouping-default.txt-Debug]": [
20962096
{
2097-
"checksum": "1a09d8d29e7fdffe5f7e7c4abe384793",
2098-
"size": 2057,
2099-
"uri": "https://{canondata_backend}/1899731/d7118ad96c050279cfcfe95a4f9577de9c404054/resource.tar.gz#test.test_pg-select_agg_gs_grouping-default.txt-Debug_/opt.yql_patched"
2097+
"checksum": "ca5a6b7b2dda9ab50d3a16f906540b6c",
2098+
"size": 1783,
2099+
"uri": "https://{canondata_backend}/1936842/7066d2b363541e6b64af6ab839e11485df1f472d/resource.tar.gz#test.test_pg-select_agg_gs_grouping-default.txt-Debug_/opt.yql_patched"
21002100
}
21012101
],
21022102
"test.test[pg-select_agg_gs_grouping-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2031,9 +2031,9 @@
20312031
],
20322032
"test.test[pg-join_brackets2-default.txt-Debug]": [
20332033
{
2034-
"checksum": "2a726748c5ccec54f860e5a2c1b0be97",
2035-
"size": 11584,
2036-
"uri": "https://{canondata_backend}/1871182/027431fe7e452cb0fbf3b9f53f15eb69aef793e3/resource.tar.gz#test.test_pg-join_brackets2-default.txt-Debug_/opt.yql_patched"
2034+
"checksum": "db58712f9297f53f8ecc04f501f45be9",
2035+
"size": 11888,
2036+
"uri": "https://{canondata_backend}/212715/536da9610d37d18c947fc7368e3720d62d90036f/resource.tar.gz#test.test_pg-join_brackets2-default.txt-Debug_/opt.yql_patched"
20372037
}
20382038
],
20392039
"test.test[pg-join_brackets2-default.txt-Plan]": [
@@ -2232,9 +2232,9 @@
22322232
],
22332233
"test.test[pg-select_join_left_one-default.txt-Debug]": [
22342234
{
2235-
"checksum": "d9307f53626d37373c8dac8bb24ef210",
2236-
"size": 3549,
2237-
"uri": "https://{canondata_backend}/1925821/4e746178682b9df3b5c4d499b609fbaf2ac5b376/resource.tar.gz#test.test_pg-select_join_left_one-default.txt-Debug_/opt.yql_patched"
2235+
"checksum": "1888b5487f5306df604737c1d87ec0a5",
2236+
"size": 3137,
2237+
"uri": "https://{canondata_backend}/212715/536da9610d37d18c947fc7368e3720d62d90036f/resource.tar.gz#test.test_pg-select_join_left_one-default.txt-Debug_/opt.yql_patched"
22382238
}
22392239
],
22402240
"test.test[pg-select_join_left_one-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part2/canondata/result.json

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -275,9 +275,9 @@
275275
],
276276
"test.test[aggregate-group_by_ru_partition_by_grouping-default.txt-Debug]": [
277277
{
278-
"checksum": "11d1fc2cfa24b26e10c896218e286158",
279-
"size": 7968,
280-
"uri": "https://{canondata_backend}/1903280/7cce39a9f54f953ff747437b2afbf4cd7340892a/resource.tar.gz#test.test_aggregate-group_by_ru_partition_by_grouping-default.txt-Debug_/opt.yql_patched"
278+
"checksum": "e70fafc8f131842282831ca56eb2170b",
279+
"size": 7988,
280+
"uri": "https://{canondata_backend}/1900335/e1d65d264295642feda5f8ea8dd5531cef75bc40/resource.tar.gz#test.test_aggregate-group_by_ru_partition_by_grouping-default.txt-Debug_/opt.yql_patched"
281281
}
282282
],
283283
"test.test[aggregate-group_by_ru_partition_by_grouping-default.txt-Plan]": [
@@ -2119,9 +2119,9 @@
21192119
],
21202120
"test.test[pg-select_join_full_one-default.txt-Debug]": [
21212121
{
2122-
"checksum": "0edb3d6848a0d8e11676a46cbae712ff",
2123-
"size": 4568,
2124-
"uri": "https://{canondata_backend}/1925842/e57e96bc0849393d23e4c2749e610a2eea073ec0/resource.tar.gz#test.test_pg-select_join_full_one-default.txt-Debug_/opt.yql_patched"
2122+
"checksum": "1c2a86e055733a692270d98d0e31c3a4",
2123+
"size": 4084,
2124+
"uri": "https://{canondata_backend}/1871182/7949078f204bbdbafbcf0efaa8e1e8cbc661f9ab/resource.tar.gz#test.test_pg-select_join_full_one-default.txt-Debug_/opt.yql_patched"
21252125
}
21262126
],
21272127
"test.test[pg-select_join_full_one-default.txt-Plan]": [
@@ -2163,9 +2163,9 @@
21632163
],
21642164
"test.test[pg-select_join_right_one-default.txt-Debug]": [
21652165
{
2166-
"checksum": "f56f23502a7de47b607eae2b9a692509",
2167-
"size": 3553,
2168-
"uri": "https://{canondata_backend}/1925842/e57e96bc0849393d23e4c2749e610a2eea073ec0/resource.tar.gz#test.test_pg-select_join_right_one-default.txt-Debug_/opt.yql_patched"
2166+
"checksum": "c7aa79af4d7bfa800205f429fe96e928",
2167+
"size": 3141,
2168+
"uri": "https://{canondata_backend}/1871182/7949078f204bbdbafbcf0efaa8e1e8cbc661f9ab/resource.tar.gz#test.test_pg-select_join_right_one-default.txt-Debug_/opt.yql_patched"
21692169
}
21702170
],
21712171
"test.test[pg-select_join_right_one-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2281,9 +2281,9 @@
22812281
],
22822282
"test.test[pg-select_join_full_equi_and_one-default.txt-Debug]": [
22832283
{
2284-
"checksum": "7b59bf677636db910c99489971ccaf56",
2285-
"size": 2402,
2286-
"uri": "https://{canondata_backend}/995452/b8907c48ceee21fc9f22e90025a80d21b7f812d0/resource.tar.gz#test.test_pg-select_join_full_equi_and_one-default.txt-Debug_/opt.yql_patched"
2284+
"checksum": "ae1e64be8eee049c0252de7d88312e13",
2285+
"size": 2068,
2286+
"uri": "https://{canondata_backend}/1599023/c48eea35b704ce8968912971b5424c6d295839a1/resource.tar.gz#test.test_pg-select_join_full_equi_and_one-default.txt-Debug_/opt.yql_patched"
22872287
}
22882288
],
22892289
"test.test[pg-select_join_full_equi_and_one-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1832,9 +1832,9 @@
18321832
],
18331833
"test.test[pg-select_agg_gs_cube-default.txt-Debug]": [
18341834
{
1835-
"checksum": "1bac7333f53acc628ab87962d55a75ff",
1836-
"size": 1362,
1837-
"uri": "https://{canondata_backend}/995452/7ec622aec5d016fd39433889930514dc0a6b650d/resource.tar.gz#test.test_pg-select_agg_gs_cube-default.txt-Debug_/opt.yql_patched"
1835+
"checksum": "e43031262078cd3099cb1d4a94fb6810",
1836+
"size": 1319,
1837+
"uri": "https://{canondata_backend}/1784826/bfd74085acfc22cb51377d828fa050c7eec78e01/resource.tar.gz#test.test_pg-select_agg_gs_cube-default.txt-Debug_/opt.yql_patched"
18381838
}
18391839
],
18401840
"test.test[pg-select_agg_gs_cube-default.txt-Plan]": [
@@ -1854,9 +1854,9 @@
18541854
],
18551855
"test.test[pg-select_agg_gs_sets-default.txt-Debug]": [
18561856
{
1857-
"checksum": "90b1e003e489951f87012b6a113e6202",
1858-
"size": 1294,
1859-
"uri": "https://{canondata_backend}/995452/7ec622aec5d016fd39433889930514dc0a6b650d/resource.tar.gz#test.test_pg-select_agg_gs_sets-default.txt-Debug_/opt.yql_patched"
1857+
"checksum": "a8b603553e0b68362e74919885dd28a4",
1858+
"size": 1137,
1859+
"uri": "https://{canondata_backend}/1784826/bfd74085acfc22cb51377d828fa050c7eec78e01/resource.tar.gz#test.test_pg-select_agg_gs_sets-default.txt-Debug_/opt.yql_patched"
18601860
}
18611861
],
18621862
"test.test[pg-select_agg_gs_sets-default.txt-Plan]": [

‎ydb/library/yql/tests/sql/hybrid_file/part1/canondata/result.json

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1751,9 +1751,9 @@
17511751
],
17521752
"test.test[pg-join_brackets2-default.txt-Debug]": [
17531753
{
1754-
"checksum": "4a701ec57a62b248d5a4f5eb4c277b89",
1755-
"size": 11583,
1756-
"uri": "https://{canondata_backend}/1871182/2666f20d4e176027b281a7b851dd96e818956ef4/resource.tar.gz#test.test_pg-join_brackets2-default.txt-Debug_/opt.yql_patched"
1754+
"checksum": "2058b53155107b107038457047899a9d",
1755+
"size": 11887,
1756+
"uri": "https://{canondata_backend}/1920236/3d3a0652b04204362f293bde5bd1db28f8645dd7/resource.tar.gz#test.test_pg-join_brackets2-default.txt-Debug_/opt.yql_patched"
17571757
}
17581758
],
17591759
"test.test[pg-join_brackets2-default.txt-Plan]": [
@@ -1905,9 +1905,9 @@
19051905
],
19061906
"test.test[pg-select_join_left_equi_and_one-default.txt-Debug]": [
19071907
{
1908-
"checksum": "1ccd7ee24ef7acfd0255b7f75ab1437b",
1909-
"size": 2261,
1910-
"uri": "https://{canondata_backend}/1903280/b3ad5a45d76b516f66899551cc71277d7e559aab/resource.tar.gz#test.test_pg-select_join_left_equi_and_one-default.txt-Debug_/opt.yql_patched"
1908+
"checksum": "83353c1216769927d1e99a6463be26e7",
1909+
"size": 1793,
1910+
"uri": "https://{canondata_backend}/1920236/3d3a0652b04204362f293bde5bd1db28f8645dd7/resource.tar.gz#test.test_pg-select_join_left_equi_and_one-default.txt-Debug_/opt.yql_patched"
19111911
}
19121912
],
19131913
"test.test[pg-select_join_left_equi_and_one-default.txt-Plan]": [

0 commit comments

Comments
 (0)