Skip to content

Commit 8792168

Browse files
zverevgenyzverevgeny
committed
Use lc-buckets optimizer by default (#13912)
1 parent ff68e99 commit 8792168

File tree

19 files changed

+253
-84
lines changed

19 files changed

+253
-84
lines changed

‎ydb/core/kqp/ut/olap/aggregations_ut.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,9 @@ Y_UNIT_TEST_SUITE(KqpOlapAggregations) {
110110
TKikimrRunner kikimr(settings);
111111

112112
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
113-
TLocalHelper(kikimr).CreateTestOlapTable();
113+
auto helper = TLocalHelper(kikimr);
114+
helper.CreateTestOlapTable();
115+
helper.SetForcedCompaction();
114116
auto tableClient = kikimr.GetTableClient();
115117

116118
{

‎ydb/core/kqp/ut/olap/dictionary_ut.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) {
4949
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 1);
5050
------
5151
SCHEMA:
52+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
53+
------
54+
SCHEMA:
5255
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
5356
------
5457
SCHEMA:
@@ -121,6 +124,9 @@ Y_UNIT_TEST_SUITE(KqpOlapDictionary) {
121124
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2$$);
122125
------
123126
SCHEMA:
127+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
128+
------
129+
SCHEMA:
124130
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
125131
------
126132
SCHEMA:

‎ydb/core/kqp/ut/olap/indexes_ut.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
5252
}
5353

5454
Y_UNIT_TEST(IndexesActualization) {
55-
auto settings = TKikimrSettings().SetWithSampleTables(false);
55+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
5656
TKikimrRunner kikimr(settings);
5757

5858
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
@@ -61,7 +61,9 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
6161
csController->SetOverrideMemoryLimitForPortionReading(1e+10);
6262
csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings());
6363

64-
TLocalHelper(kikimr).CreateTestOlapTable();
64+
auto helper = TLocalHelper(kikimr);
65+
helper.CreateTestOlapTable();
66+
helper.SetForcedCompaction();
6567
auto tableClient = kikimr.GetTableClient();
6668

6769
Tests::NCommon::TLoggerInit(kikimr)

‎ydb/core/kqp/ut/olap/json_ut.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
3939
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2$$);
4040
------
4141
SCHEMA:
42+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
43+
------
44+
SCHEMA:
4245
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
4346
------
4447
SCHEMA:
@@ -529,6 +532,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
529532
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
530533
------
531534
SCHEMA:
535+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
536+
------
537+
SCHEMA:
532538
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
533539
------
534540
SCHEMA:
@@ -843,6 +849,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
843849
PARTITION BY HASH(Col1)
844850
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
845851
------
852+
SCHEMA:
853+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
854+
------
846855
DATA:
847856
REPLACE INTO `/Root/ColumnTable` (Col1, Col2) VALUES(1u, JsonDocument('{"a" : "a1"}')), (2u, JsonDocument('{"a" : "a2"}')),
848857
(3u, JsonDocument('{"b" : "b3"}')), (4u, JsonDocument('{"b" : "b4", "a" : "a4"}'))
@@ -885,6 +894,9 @@ Y_UNIT_TEST_SUITE(KqpOlapJson) {
885894
WITH (STORE = COLUMN, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = $$1|2|10$$);
886895
------
887896
SCHEMA:
897+
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)
898+
------
899+
SCHEMA:
888900
ALTER OBJECT `/Root/ColumnTable` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `SCAN_READER_POLICY_NAME`=`SIMPLE`)
889901
------
890902
SCHEMA:

‎ydb/core/kqp/ut/olap/kqp_olap_ut.cpp

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2930,6 +2930,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
29302930
Y_UNIT_TEST(BulkUpsertUpdate) {
29312931
TKikimrSettings runnerSettings;
29322932
runnerSettings.WithSampleTables = false;
2933+
runnerSettings.SetColumnShardAlterObjectEnabled(true);
29332934
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
29342935
TTestHelper testHelper(runnerSettings);
29352936

@@ -2941,15 +2942,15 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
29412942
TTestHelper::TColumnTable testTable;
29422943
testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema);
29432944
testHelper.CreateTable(testTable);
2945+
{
2946+
auto result = testHelper.GetSession().ExecuteSchemeQuery("ALTER OBJECT `/Root/ColumnTableTest` (TYPE TABLE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`l-buckets`)").GetValueSync();
2947+
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
2948+
}
29442949
{
29452950
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));
29462951
tableInserter.AddRow().Add(1).Add(10);
29472952
testHelper.BulkUpsert(testTable, tableInserter);
29482953
}
2949-
// while (csController->GetCompactionFinishedCounter().Val() < 1) {
2950-
// Cout << "Wait compaction..." << Endl;
2951-
// Sleep(TDuration::Seconds(2));
2952-
// }
29532954
testHelper.ReadData("SELECT value FROM `/Root/ColumnTableTest` WHERE id = 1", "[[10]]");
29542955
{
29552956
TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema));

‎ydb/core/kqp/ut/olap/sys_view_ut.cpp

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ namespace NKikimr::NKqp {
1414
Y_UNIT_TEST_SUITE(KqpOlapSysView) {
1515
Y_UNIT_TEST(StatsSysView) {
1616
auto settings = TKikimrSettings()
17-
.SetWithSampleTables(false);
17+
.SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
1818
TKikimrRunner kikimr(settings);
1919

2020
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>();
21-
TLocalHelper(kikimr).CreateTestOlapTable();
21+
auto helper = TLocalHelper(kikimr);
22+
helper.CreateTestOlapTable();
23+
helper.SetForcedCompaction();
2224
for (ui64 i = 0; i < 100; ++i) {
2325
WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000 + i * 10000, 1000);
2426
}
@@ -633,13 +635,14 @@ Y_UNIT_TEST_SUITE(KqpOlapSysView) {
633635
}
634636

635637
Y_UNIT_TEST(StatsSysViewAggregation) {
636-
auto settings = TKikimrSettings().SetWithSampleTables(false);
638+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
637639
TKikimrRunner kikimr(settings);
638640
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NOlap::TWaitCompactionController>();
639-
640-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_1");
641-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_2");
642-
TLocalHelper(kikimr.GetTestServer()).CreateTestOlapTable("olapTable_3");
641+
TLocalHelper helper(kikimr.GetTestServer());
642+
helper.CreateTestOlapTable("olapTable_1");
643+
helper.CreateTestOlapTable("olapTable_2");
644+
helper.CreateTestOlapTable("olapTable_3");
645+
helper.SetForcedCompaction();
643646

644647
for (ui64 i = 0; i < 100; ++i) {
645648
WriteTestData(kikimr, "/Root/olapStore/olapTable_1", 0, 1000000 + i * 10000, 1000);

‎ydb/core/kqp/ut/olap/write_ut.cpp

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
4545

4646
auto settings = TKikimrSettings().SetWithSampleTables(false);
4747
TKikimrRunner kikimr(settings);
48-
TLocalHelper(kikimr).CreateTestOlapTable();
48+
auto helper = TLocalHelper(kikimr);
49+
helper.CreateTestOlapTable();
50+
helper.SetForcedCompaction();
51+
4952
Tests::NCommon::TLoggerInit(kikimr)
5053
.SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS")
5154
.SetPriority(NActors::NLog::PRI_DEBUG)
@@ -195,13 +198,15 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
195198
}
196199

197200
Y_UNIT_TEST(MultiWriteInTime) {
198-
auto settings = TKikimrSettings().SetWithSampleTables(false);
201+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
199202
settings.AppConfig.MutableColumnShardConfig()->SetWritingBufferDurationMs(15000);
200203
TKikimrRunner kikimr(settings);
201204
Tests::NCommon::TLoggerInit(kikimr).Initialize();
202205
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TReadOnlyController>();
203206
TTypedLocalHelper helper("Utf8", kikimr);
204207
helper.CreateTestOlapTable();
208+
helper.SetForcedCompaction();
209+
205210
auto writeSession = helper.StartWriting("/Root/olapStore/olapTable");
206211
writeSession.FillTable("field", NArrow::NConstruction::TStringPoolFiller(1, 1, "aaa", 1), 0, 800000);
207212
Sleep(TDuration::Seconds(1));
@@ -250,13 +255,14 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
250255
}
251256

252257
Y_UNIT_TEST(MultiWriteInTimeDiffSchemas) {
253-
auto settings = TKikimrSettings().SetWithSampleTables(false);
258+
auto settings = TKikimrSettings().SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
254259
settings.AppConfig.MutableColumnShardConfig()->SetWritingBufferDurationMs(15000);
255260
TKikimrRunner kikimr(settings);
256261
Tests::NCommon::TLoggerInit(kikimr).Initialize();
257262
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TReadOnlyController>();
258263
TTypedLocalHelper helper("Utf8", "Utf8", kikimr);
259264
helper.CreateTestOlapTable();
265+
helper.SetForcedCompaction();
260266
auto writeGuard = helper.StartWriting("/Root/olapStore/olapTable");
261267
writeGuard.FillTable("field", NArrow::NConstruction::TStringPoolFiller(1, 1, "aaa", 1), 0, 800000);
262268
Sleep(TDuration::Seconds(1));
@@ -313,9 +319,11 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
313319
NKikimrConfig::TAppConfig appConfig;
314320
appConfig.MutableTableServiceConfig()->SetEnableOlapSink(true);
315321

316-
auto settings = TKikimrSettings().SetAppConfig(appConfig).SetWithSampleTables(false);
322+
auto settings = TKikimrSettings().SetAppConfig(appConfig).SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true);
317323
TKikimrRunner kikimr(settings);
318-
TLocalHelper(kikimr).CreateTestOlapTable();
324+
auto helper = TLocalHelper(kikimr);
325+
helper.CreateTestOlapTable();
326+
helper.SetForcedCompaction();
319327
Tests::NCommon::TLoggerInit(kikimr)
320328
.SetComponents({ NKikimrServices::TX_COLUMNSHARD, NKikimrServices::TX_COLUMNSHARD_BLOBS }, "CS")
321329
.SetPriority(NActors::NLog::PRI_DEBUG)

‎ydb/core/testlib/cs_helper.cpp

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ void THelperSchemaless::ExecuteModifyScheme(NKikimrSchemeOp::TModifyScheme& modi
2424
TActorId sender = Server.GetRuntime()->AllocateEdgeActor();
2525
Server.GetRuntime()->Send(new IEventHandle(MakeTxProxyID(), sender, request.release()));
2626
auto ev = Server.GetRuntime()->GrabEdgeEventRethrow<TEvTxUserProxy::TEvProposeTransactionStatus>(sender);
27-
Cerr << ev->Get()->Record.DebugString() << Endl;
2827
auto status = ev->Get()->Record.GetStatus();
2928
ui64 txId = ev->Get()->Record.GetTxId();
3029
UNIT_ASSERT(status != TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ExecError);
@@ -100,7 +99,6 @@ void THelperSchemaless::SendDataViaActorSystem(TString testTable, ui64 pathIdBeg
10099
SendDataViaActorSystem(testTable, batch);
101100
}
102101

103-
//
104102

105103
std::shared_ptr<arrow::Schema> THelper::GetArrowSchema() const {
106104
std::vector<std::shared_ptr<arrow::Field>> fields;
@@ -176,6 +174,29 @@ std::shared_ptr<arrow::RecordBatch> THelper::TestArrowBatch(ui64 pathIdBegin, ui
176174

177175
}
178176

177+
void THelper::SetForcedCompaction(const TString& storeName) {
178+
//In some tests we expect, that a compaction will start immidiately
179+
//For now, we use l-bucket optimizer for this purpose
180+
//In the future it should be replaced with lc-bucket or more sophisticated compaction optimizer planner
181+
auto request = std::make_unique<TEvTxUserProxy::TEvProposeTransaction>();
182+
request->Record.SetExecTimeoutPeriod(Max<ui64>());
183+
NKikimrSchemeOp::TModifyScheme modyfySchemeOp;
184+
modyfySchemeOp.SetOperationType(NKikimrSchemeOp::EOperationType::ESchemeOpAlterColumnStore);
185+
modyfySchemeOp.SetWorkingDir(ROOT_PATH);
186+
NKikimrSchemeOp::TAlterColumnStore* alterColumnStore = modyfySchemeOp.MutableAlterColumnStore();
187+
alterColumnStore->SetName(storeName);
188+
auto schemaPreset = alterColumnStore->AddAlterSchemaPresets();
189+
schemaPreset->SetName("default");
190+
auto schemaOptions = schemaPreset->MutableAlterSchema()->MutableOptions();
191+
schemaOptions->SetSchemeNeedActualization(false);
192+
auto plannerConstructot =schemaOptions->MutableCompactionPlannerConstructor();
193+
plannerConstructot->SetClassName("l-buckets");
194+
*plannerConstructot->MutableLBuckets() = NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TLOptimizer{};
195+
196+
ExecuteModifyScheme(modyfySchemeOp);
197+
}
198+
199+
179200
TString THelper::GetTestTableSchema() const {
180201
TStringBuilder sb;
181202
sb << R"(Columns{ Name: "timestamp" Type : "Timestamp" NotNull : true })";

‎ydb/core/testlib/cs_helper.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,8 @@ class THelper: public THelperSchemaless {
5454
return *this;
5555
}
5656

57+
void SetForcedCompaction(const TString& storeName = "olapStore");
58+
5759
static constexpr const char * PROTO_SCHEMA = R"(
5860
Columns { Name: "timestamp" Type: "Timestamp" NotNull: true }
5961
Columns { Name: "resource_id" Type: "Utf8" DataAccessorConstructor{ ClassName: "SPARSED" }}

‎ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,17 +178,24 @@ class IOptimizerPlanner {
178178

179179
class IOptimizerPlannerConstructor {
180180
public:
181+
enum class EOptimizerStrategy {
182+
Default, //use One Layer levels to avoid portion intersections
183+
Logs, // use Zero Levels only for performance
184+
LogsInStore
185+
};
181186
class TBuildContext {
182187
private:
183188
YDB_READONLY_DEF(TInternalPathId, PathId);
184189
YDB_READONLY_DEF(std::shared_ptr<IStoragesManager>, Storages);
185190
YDB_READONLY_DEF(std::shared_ptr<arrow::Schema>, PKSchema);
191+
YDB_READONLY_DEF(EOptimizerStrategy, DefaultStrategy);
186192

187193
public:
188194
TBuildContext(const TInternalPathId pathId, const std::shared_ptr<IStoragesManager>& storages, const std::shared_ptr<arrow::Schema>& pkSchema)
189195
: PathId(pathId)
190196
, Storages(storages)
191-
, PKSchema(pkSchema) {
197+
, PKSchema(pkSchema)
198+
, DefaultStrategy(EOptimizerStrategy::Default) { //TODO configure me via DDL
192199
}
193200
};
194201

@@ -204,7 +211,7 @@ class IOptimizerPlannerConstructor {
204211

205212
public:
206213
static std::shared_ptr<IOptimizerPlannerConstructor> BuildDefault() {
207-
auto result = TFactory::MakeHolder("l-buckets");
214+
auto result = TFactory::MakeHolder("lc-buckets");
208215
AFL_VERIFY(!!result);
209216
return std::shared_ptr<IOptimizerPlannerConstructor>(result.Release());
210217
}

0 commit comments

Comments
 (0)