Skip to content

Commit 673dab6

Browse files
zverevgenyzverevgeny
authored
Lc buckets by default (#19229)
2 parents 59239bf + 3878fbc commit 673dab6

File tree

25 files changed

+295
-134
lines changed

25 files changed

+295
-134
lines changed

‎ydb/core/kqp/ut/olap/aggregations_ut.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,9 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) {
110110
TKikimrRunner kikimr(settings);
111111

112112
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
113-
TLocalHelper(kikimr).CreateTestOlapTable();
113+
auto helper = TLocalHelper(kikimr);
114+
helper.CreateTestOlapTable();
115+
helper.SetForcedCompaction();
114116
auto tableClient = kikimr.GetTableClient();
115117

116118
{

‎ydb/core/kqp/ut/olap/clickbench_ut.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ Y_UNIT_TEST_SUITE(KqpOlapClickbench) {
2020
{}
2121

2222
void CreateClickBenchTable(TString tableName = "benchTable", ui32 shardsCount = 4) {
23-
TActorId sender = Server.GetRuntime()->AllocateEdgeActor();
24-
25-
TBase::CreateTestOlapTable(sender, "", Sprintf(R"(
23+
TBase::CreateTestOlapTable( "", Sprintf(R"(
2624
Name: "%s"
2725
ColumnShardCount: %d
2826
Schema {

‎ydb/core/kqp/ut/olap/dictionary_ut.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) {
4949
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1);
5050
------
5151
SCHEMA:
52+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
53+
------
54+
SCHEMA:
5255
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
5356
------
5457
SCHEMA:
@@ -121,6 +124,9 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) {
121124
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2$$);
122125
------
123126
SCHEMA:
127+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
128+
------
129+
SCHEMA:
124130
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
125131
------
126132
SCHEMA:

‎ydb/core/kqp/ut/olap/helpers/local.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
namespace NKikimr::NKqp {
44

55
void TTableWithNullsHelper::CreateTableWithNulls(TString tableName /*= "tableWithNulls"*/, ui32 shardsCount /*= 4*/) {
6-
TActorId sender = Server.GetRuntime()->AllocateEdgeActor();
7-
8-
TBase::CreateTestOlapTable(sender, "", Sprintf(R"(
6+
TBase::CreateTestOlapTable("", Sprintf(R"(
97
Name: "%s"
108
ColumnShardCount: %d
119
Schema {

‎ydb/core/kqp/ut/olap/indexes_ut.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
5252
}
5353

5454
Y_UNIT_TEST(IndexesActualization) {
55-
auto settings = TKikimrSettings().SetWithSampleTables(false);
55+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
5656
TKikimrRunner kikimr(settings);
5757

5858
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
@@ -61,7 +61,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
6161
csController->SetOverrideMemoryLimitForPortionReading(1e+10);
6262
csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings());
6363

64-
TLocalHelper(kikimr).CreateTestOlapTable();
64+
auto helper = TLocalHelper(kikimr);
65+
helper.CreateTestOlapTable();
66+
helper.SetForcedCompaction();
6567
auto tableClient = kikimr.GetTableClient();
6668

6769
Tests::NCommon::TLoggerInit(kikimr)

‎ydb/core/kqp/ut/olap/json_ut.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
3939
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2$$);
4040
------
4141
SCHEMA:
42+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
43+
------
44+
SCHEMA:
4245
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
4346
------
4447
SCHEMA:
@@ -529,6 +532,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
529532
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
530533
------
531534
SCHEMA:
535+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
536+
------
537+
SCHEMA:
532538
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
533539
------
534540
SCHEMA:
@@ -843,6 +849,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
843849
PARTITION BY HASH(Col1)
844850
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
845851
------
852+
SCHEMA:
853+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
854+
------
846855
DATA:
847856
REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES(1u, JsonDocument('{"a" : "a1"}')), (2u, JsonDocument('{"a" : "a2"}')),
848857
(3u, JsonDocument('{"b" : "b3"}')), (4u, JsonDocument('{"b" : "b4", "a" : "a4"}'))
@@ -885,6 +894,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
885894
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
886895
------
887896
SCHEMA:
897+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
898+
------
899+
SCHEMA:
888900
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
889901
------
890902
SCHEMA:

‎ydb/core/kqp/ut/olap/kqp_olap_ut.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2930,6 +2930,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
29302930
Y_UNIT_TEST(BulkUpsertUpdate) {
29312931
TKikimrSettings runnerSettings;
29322932
runnerSettings.WithSampleTables = false;
2933+
runnerSettings.SetColumnShardAlterObjectEnabled(true);
29332934
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
29342935
TTestHelper testHelper(runnerSettings);
29352936

@@ -2941,15 +2942,15 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
29412942
TTestHelper::TColumnTable testTable;
29422943
testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema);
29432944
testHelper.CreateTable(testTable);
2945+
{
2946+
auto result = testHelper.GetSession().ExecuteSchemeQuery("ALTER OBJECT `/Root/ColumnTableTest` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)").GetValueSync();
2947+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
2948+
}
29442949
{
29452950
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
29462951
tableInserter.AddRow().Add(1).Add(10);
29472952
testHelper.BulkUpsert(testTable, tableInserter);
29482953
}
2949-
// while (csController->GetCompactionFinishedCounter().Val() < 1) {
2950-
// Cout << "Wait compaction..." << Endl;
2951-
// Sleep(TDuration::Seconds(2));
2952-
// }
29532954
testHelper.ReadData("SELECT value FROM `/Root/ColumnTableTest` WHERE id = 1", "[[10]]");
29542955
{
29552956
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));

‎ydb/core/kqp/ut/olap/sys_view_ut.cpp

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ namespace NKikimr::NKqp {
1414
Y_UNIT_TEST_SUITE(KqpOlapSysView) {
1515
Y_UNIT_TEST(StatsSysView) {
1616
auto settings = TKikimrSettings()
17-
.SetWithSampleTables(false);
17+
.SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
1818
TKikimrRunner kikimr(settings);
1919

2020
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>();
21-
TLocalHelper(kikimr).CreateTestOlapTable();
21+
auto helper = TLocalHelper(kikimr);
22+
helper.CreateTestOlapTable();
23+
helper.SetForcedCompaction();
2224
for (ui64 i = 0; i < 100; ++i) {
2325
WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000 + i * 10000, 1000);
2426
}
@@ -633,13 +635,14 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) {
633635
}
634636

635637
Y_UNIT_TEST(StatsSysViewAggregation) {
636-
auto settings = TKikimrSettings().SetWithSampleTables(false);
638+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
637639
TKikimrRunner kikimr(settings);
638640
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>();
639-
640-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_1");
641-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_2");
642-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_3");
641+
TLocalHelper helper(kikimr.GetTestServer());
642+
helper.CreateTestOlapTable("olapTable_1");
643+
helper.CreateTestOlapTable("olapTable_2");
644+
helper.CreateTestOlapTable("olapTable_3");
645+
helper.SetForcedCompaction();
643646

644647
for (ui64 i = 0; i < 100; ++i) {
645648
WriteTestData(kikimr, "/Root/olapStore/olapTable_1", 0, 1000000 + i * 10000, 1000);

‎ydb/core/kqp/ut/olap/write_ut.cpp

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
4545

4646
auto settings = TKikimrSettings().SetWithSampleTables(false);
4747
TKikimrRunner kikimr(settings);
48-
TLocalHelper(kikimr).CreateTestOlapTable();
48+
auto helper = TLocalHelper(kikimr);
49+
helper.CreateTestOlapTable();
50+
helper.SetForcedCompaction();
51+
4952
Tests::NCommon::TLoggerInit(kikimr)
5053
.SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS")
5154
.SetPriority(NActors::NLog::PRI_DEBUG)
@@ -195,13 +198,15 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
195198
}
196199

197200
Y_UNIT_TEST(MultiWriteInTime) {
198-
auto settings = TKikimrSettings().SetWithSampleTables(false);
201+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
199202
settings.AppConfig.MutableColumnShardConfig()->SetWritingBufferDurationMs(15000);
200203
TKikimrRunner kikimr(settings);
201204
Tests::NCommon::TLoggerInit(kikimr).Initialize();
202205
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TReadOnlyController>();
203206
TTypedLocalHelper helper("Utf8", kikimr);
204207
helper.CreateTestOlapTable();
208+
helper.SetForcedCompaction();
209+
205210
auto writeSession = helper.StartWriting("/Root/olapStore/olapTable");
206211
writeSession.FillTable("field", NArrow::NConstruction::TStringPoolFiller(1, 1, "aaa", 1), 0, 800000);
207212
Sleep(TDuration::Seconds(1));
@@ -250,13 +255,14 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
250255
}
251256

252257
Y_UNIT_TEST(MultiWriteInTimeDiffSchemas) {
253-
auto settings = TKikimrSettings().SetWithSampleTables(false);
258+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
254259
settings.AppConfig.MutableColumnShardConfig()->SetWritingBufferDurationMs(15000);
255260
TKikimrRunner kikimr(settings);
256261
Tests::NCommon::TLoggerInit(kikimr).Initialize();
257262
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TReadOnlyController>();
258263
TTypedLocalHelper helper("Utf8", "Utf8", kikimr);
259264
helper.CreateTestOlapTable();
265+
helper.SetForcedCompaction();
260266
auto writeGuard = helper.StartWriting("/Root/olapStore/olapTable");
261267
writeGuard.FillTable("field", NArrow::NConstruction::TStringPoolFiller(1, 1, "aaa", 1), 0, 800000);
262268
Sleep(TDuration::Seconds(1));
@@ -313,9 +319,11 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
313319
NKikimrConfig::TAppConfig appConfig;
314320
appConfig.MutableTableServiceConfig()->SetEnableOlapSink(true);
315321

316-
auto settings = TKikimrSettings().SetAppConfig(appConfig).SetWithSampleTables(false);
322+
auto settings = TKikimrSettings().SetAppConfig(appConfig).SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
317323
TKikimrRunner kikimr(settings);
318-
TLocalHelper(kikimr).CreateTestOlapTable();
324+
auto helper = TLocalHelper(kikimr);
325+
helper.CreateTestOlapTable();
326+
helper.SetForcedCompaction();
319327
Tests::NCommon::TLoggerInit(kikimr)
320328
.SetComponents({ NKikimrServices::TX_COLUMNSHARD, NKikimrServices::TX_COLUMNSHARD_BLOBS }, "CS")
321329
.SetPriority(NActors::NLog::PRI_DEBUG)

‎ydb/core/statistics/ut_common/ut_common.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,11 @@ void CreateColumnStoreTable(TTestEnv& env, const TString& databaseName, const TS
289289
)", fullTableName.c_str()));
290290
runtime.SimulateSleep(TDuration::Seconds(1));
291291

292+
ExecuteYqlScript(env, Sprintf(R"(
293+
ALTER OBJECT `%s` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`);
294+
)", fullTableName.c_str()));
295+
runtime.SimulateSleep(TDuration::Seconds(1));
296+
292297
ExecuteYqlScript(env, Sprintf(R"(
293298
ALTER OBJECT `%s` (TYPE TABLE) SET (ACTION=UPSERT_INDEX, NAME=cms_value, TYPE=COUNT_MIN_SKETCH,
294299
FEATURES=`{"column_names" : ['Value']}`);

0 commit comments

Comments
 (0)