Skip to content

Commit 4a55ea4

Browse files
authored
chore: default enable aggregate/sort/join spill (#15038)
* chore: default enable aggregate/sort/join spill * fix the explain sort/window.test for the spill enabled
1 parent 722ae22 commit 4a55ea4

File tree

3 files changed

+98
-3
lines changed

3 files changed

+98
-3
lines changed

src/query/settings/src/settings_default.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ impl DefaultSettings {
273273
range: Some(SettingRange::Numeric(0..=1)),
274274
}),
275275
("join_spilling_memory_ratio", DefaultSettingValue {
276-
value: UserSettingValue::UInt64(0),
276+
value: UserSettingValue::UInt64(60),
277277
desc: "Sets the maximum memory ratio in bytes that hash join can use before spilling data to storage during query execution, 0 is unlimited",
278278
mode: SettingMode::Both,
279279
range: Some(SettingRange::Numeric(0..=100)),
@@ -412,7 +412,7 @@ impl DefaultSettings {
412412
range: Some(SettingRange::Numeric(0..=u64::MAX)),
413413
}),
414414
("aggregate_spilling_memory_ratio", DefaultSettingValue {
415-
value: UserSettingValue::UInt64(0),
415+
value: UserSettingValue::UInt64(60),
416416
desc: "Sets the maximum memory ratio in bytes that an aggregator can use before spilling data to storage during query execution.",
417417
mode: SettingMode::Both,
418418
range: Some(SettingRange::Numeric(0..=100)),
@@ -424,7 +424,7 @@ impl DefaultSettings {
424424
range: Some(SettingRange::Numeric(0..=u64::MAX)),
425425
}),
426426
("sort_spilling_memory_ratio", DefaultSettingValue {
427-
value: UserSettingValue::UInt64(0),
427+
value: UserSettingValue::UInt64(60),
428428
desc: "Sets the maximum memory ratio in bytes that a sorter can use before spilling data to storage during query execution.",
429429
mode: SettingMode::Both,
430430
range: Some(SettingRange::Numeric(0..=100)),

tests/sqllogictests/suites/mode/standalone/explain/sort.test

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ Sort
6767
statement ok
6868
set max_threads = 4;
6969

70+
statement ok
71+
set sort_spilling_memory_ratio = 0;
7072

7173
# Sort without pre-projection
7274
query T
@@ -81,6 +83,25 @@ CompoundBlockOperator(Project) × 1 processor
8183
SyncReadParquetDataSource × 1 processor
8284

8385

86+
# Sort spilling
87+
statement ok
88+
set sort_spilling_memory_ratio = 60;
89+
90+
query T
91+
explain pipeline select a, b from t1 order by a;
92+
----
93+
CompoundBlockOperator(Project) × 1 processor
94+
Merge (TransformSortSpill × 4 processors) to (CompoundBlockOperator(Project) × 1)
95+
TransformSortSpill × 4 processors
96+
TransformSortMerge × 4 processors
97+
SortPartialTransform × 4 processors
98+
Merge (DeserializeDataTransform × 1 processor) to (SortPartialTransform × 4)
99+
DeserializeDataTransform × 1 processor
100+
SyncReadParquetDataSource × 1 processor
101+
102+
statement ok
103+
set sort_spilling_memory_ratio = 0;
104+
84105
# Sort with pre-projection
85106
query T
86107
explain pipeline select a + 1, b from t1 order by a + 1;
@@ -95,5 +116,22 @@ CompoundBlockOperator(Project) × 1 processor
95116
SyncReadParquetDataSource × 1 processor
96117

97118

119+
# Sort spilling
120+
statement ok
121+
set sort_spilling_memory_ratio = 60;
122+
123+
query T
124+
explain pipeline select a + 1, b from t1 order by a + 1;
125+
----
126+
CompoundBlockOperator(Project) × 1 processor
127+
Merge (TransformSortSpill × 4 processors) to (CompoundBlockOperator(Project) × 1)
128+
TransformSortSpill × 4 processors
129+
TransformSortMerge × 4 processors
130+
SortPartialTransform × 4 processors
131+
Merge (CompoundBlockOperator(Map) × 1 processor) to (SortPartialTransform × 4)
132+
CompoundBlockOperator(Map) × 1 processor
133+
DeserializeDataTransform × 1 processor
134+
SyncReadParquetDataSource × 1 processor
135+
98136
statement ok
99137
drop table if exists t1;

tests/sqllogictests/suites/mode/standalone/explain/window.test

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ Sort
3636
statement ok
3737
set max_threads=4;
3838

39+
# Disable sort spilling
40+
statement ok
41+
set sort_spilling_memory_ratio = 0;
42+
3943
query T
4044
explain pipeline SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname ORDER BY empno) FROM empsalary ORDER BY depname, empno;
4145
----
@@ -52,6 +56,30 @@ CompoundBlockOperator(Project) × 1 processor
5256
DeserializeDataTransform × 1 processor
5357
SyncReadParquetDataSource × 1 processor
5458

59+
60+
# Enable sort spilling
61+
statement ok
62+
set sort_spilling_memory_ratio = 60;
63+
64+
query T
65+
explain pipeline SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname ORDER BY empno) FROM empsalary ORDER BY depname, empno;
66+
----
67+
CompoundBlockOperator(Project) × 1 processor
68+
Merge (TransformSortSpill × 4 processors) to (CompoundBlockOperator(Project) × 1)
69+
TransformSortSpill × 4 processors
70+
TransformSortMerge × 4 processors
71+
SortPartialTransform × 4 processors
72+
Merge (Transform Window × 1 processor) to (SortPartialTransform × 4)
73+
Transform Window × 1 processor
74+
Merge (TransformSortSpill × 4 processors) to (Transform Window × 1)
75+
TransformSortSpill × 4 processors
76+
TransformSortMerge × 4 processors
77+
SortPartialTransform × 4 processors
78+
Merge (DeserializeDataTransform × 1 processor) to (SortPartialTransform × 4)
79+
DeserializeDataTransform × 1 processor
80+
SyncReadParquetDataSource × 1 processor
81+
82+
5583
statement ok
5684
DROP TABLE IF EXISTS Test
5785

@@ -292,6 +320,11 @@ drop table if exists t
292320
statement ok
293321
create table t (a int, b int, c string, d int, e int, f string)
294322

323+
324+
# Disable sort spilling
325+
statement ok
326+
set sort_spilling_memory_ratio = 0;
327+
295328
# range frame (can not push down limit)
296329
query T
297330
explain pipeline select a, sum(a) over (partition by a order by a desc) from t limit 3
@@ -306,6 +339,30 @@ CompoundBlockOperator(Project) × 1 processor
306339
DeserializeDataTransform × 1 processor
307340
SyncReadParquetDataSource × 1 processor
308341

342+
# Enable sort spilling
343+
statement ok
344+
set sort_spilling_memory_ratio = 60;
345+
346+
# range frame (can not push down limit)
347+
query T
348+
explain pipeline select a, sum(a) over (partition by a order by a desc) from t limit 3
349+
----
350+
CompoundBlockOperator(Project) × 1 processor
351+
LimitTransform × 1 processor
352+
Transform Window × 1 processor
353+
Merge (TransformSortSpill × 4 processors) to (Transform Window × 1)
354+
TransformSortSpill × 4 processors
355+
TransformSortMerge × 4 processors
356+
SortPartialTransform × 4 processors
357+
Merge (DeserializeDataTransform × 1 processor) to (SortPartialTransform × 4)
358+
DeserializeDataTransform × 1 processor
359+
SyncReadParquetDataSource × 1 processor
360+
361+
362+
# Disable sort spilling
363+
statement ok
364+
set sort_spilling_memory_ratio = 0;
365+
309366
# range frame with ranking function (can push down limit)
310367
query T
311368
explain pipeline select a, dense_rank() over (partition by a order by a desc) from t limit 3

0 commit comments

Comments
 (0)