Skip to content

Commit 3e3ffd1

Browse files
swalrus1swalrus1
authored
Add column shard Controller parameters to AppData (#8000)
1 parent 531e359 commit 3e3ffd1

File tree

31 files changed

+223
-165
lines changed

31 files changed

+223
-165
lines changed

‎ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -93,8 +93,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
9393
, Controller(NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>()) {
9494
Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable);
9595
Controller->SetExpectedShardsCount(ShardsCount);
96-
Controller->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
97-
Controller->SetReadTimeoutClean(TDuration::Seconds(1));
96+
Controller->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
97+
Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1));
9898

9999
Tests::NCommon::TLoggerInit(Kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS").Initialize();
100100

@@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
111111
}
112112

113113
void WaitNormalization() {
114-
Controller->SetReadTimeoutClean(TDuration::Seconds(1));
114+
Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1));
115115
Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force);
116116
const auto start = TInstant::Now();
117117
while (!Controller->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(30)) {
@@ -120,11 +120,11 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
120120
}
121121
AFL_VERIFY(Controller->IsTrivialLinks());
122122
Controller->CheckInvariants();
123-
Controller->SetReadTimeoutClean(TDuration::Minutes(5));
123+
Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5));
124124
}
125125

126126
void Execute(const ui64 destinationIdx, const std::vector<ui64>& sourceIdxs, const bool move, const NOlap::TSnapshot& snapshot, const std::set<ui64>& pathIdxs) {
127-
Controller->SetReadTimeoutClean(TDuration::Seconds(1));
127+
Controller->SetOverrideReadTimeoutClean(TDuration::Seconds(1));
128128
AFL_VERIFY(destinationIdx < ShardIds.size());
129129
const ui64 destination = ShardIds[destinationIdx];
130130
std::vector<ui64> sources;
@@ -192,7 +192,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
192192
CSTransferStatus->Reset();
193193
AFL_VERIFY(!Controller->IsTrivialLinks());
194194
Controller->CheckInvariants();
195-
Controller->SetReadTimeoutClean(TDuration::Minutes(5));
195+
Controller->SetOverrideReadTimeoutClean(TDuration::Minutes(5));
196196
}
197197
};
198198
Y_UNIT_TEST(BlobsSharingSplit1_1) {
@@ -318,8 +318,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
318318

319319
void Execute() {
320320
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
321-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
322-
csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1));
321+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
322+
csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1));
323323
csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30);
324324

325325
TLocalHelper(Kikimr).SetShardingMethod(ShardingType).CreateTestOlapTable("olapTable", "olapStore", 24, 4);

‎ydb/core/kqp/ut/olap/indexes_ut.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
1717
TKikimrRunner kikimr(settings);
1818

1919
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
20-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
21-
csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1));
20+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
21+
csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1));
2222
csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30);
2323

2424
TLocalHelper(kikimr).CreateTestOlapTable();
@@ -111,7 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
111111
TKikimrRunner kikimr(settings);
112112

113113
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
114-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
114+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
115115

116116
TLocalHelper(kikimr).CreateTestOlapTable();
117117
auto tableClient = kikimr.GetTableClient();

‎ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) {
2121

2222
class TOlapStatsController : public NYDBTest::NColumnShard::TController {
2323
public:
24-
TDuration GetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override {
24+
TDuration DoGetPeriodicWakeupActivationPeriod(const TDuration /*defaultValue*/) const override {
2525
return TDuration::MilliSeconds(10);
2626
}
27-
TDuration GetStatsReportInterval(const TDuration /*defaultValue*/) const override {
27+
TDuration DoGetStatsReportInterval(const TDuration /*defaultValue*/) const override {
2828
return TDuration::MilliSeconds(10);
2929
}
3030
};

‎ydb/core/kqp/ut/olap/kqp_olap_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2695,8 +2695,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
26952695
TLocalHelper testHelper(kikimr);
26962696

26972697
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
2698-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
2699-
csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1));
2698+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
2699+
csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1));
27002700
csController->SetOverrideReduceMemoryIntervalLimit(1LLU << 30);
27012701
csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation);
27022702

‎ydb/core/kqp/ut/olap/sparsed_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) {
9595
void Execute() {
9696
CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation);
9797
CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction);
98-
CSController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100));
98+
CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100));
9999

100100
Tests::NCommon::TLoggerInit(Kikimr).Initialize();
101101
TTypedLocalHelper helper("Utf8", Kikimr);

‎ydb/core/kqp/ut/olap/write_ut.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
1515
Y_UNIT_TEST(TierDraftsGC) {
1616
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>();
1717
csController->SetIndexWriteControllerEnabled(false);
18-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
18+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
1919
Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters();
2020

2121
auto settings = TKikimrSettings()
@@ -50,7 +50,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
5050
Y_UNIT_TEST(TierDraftsGCWithRestart) {
5151
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>();
5252
csController->SetIndexWriteControllerEnabled(false);
53-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1000));
53+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1000));
5454
csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC);
5555
Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters();
5656

@@ -133,7 +133,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
133133

134134
Y_UNIT_TEST(WriteDeleteCleanGC) {
135135
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>();
136-
csController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100));
136+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100));
137137
csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC);
138138
Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters();
139139

@@ -176,7 +176,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
176176
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
177177
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
178178
}
179-
csController->SetReadTimeoutClean(TDuration::Zero());
179+
csController->SetOverrideReadTimeoutClean(TDuration::Zero());
180180
csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC);
181181
{
182182
const TInstant start = TInstant::Now();

‎ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3265,8 +3265,8 @@ Y_UNIT_TEST_SUITE(KqpQueryService) {
32653265
auto session = Kikimr->GetTableClient().CreateSession().GetValueSync().GetSession();
32663266

32673267
auto csController = NYDBTest::TControllers::RegisterCSControllerGuard<NYDBTest::NColumnShard::TController>();
3268-
csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1));
3269-
csController->SetLagForCompactionBeforeTierings(TDuration::Seconds(1));
3268+
csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
3269+
csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1));
32703270
csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation);
32713271

32723272
const TString query = Sprintf(R"(

‎ydb/core/protos/config.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1553,6 +1553,12 @@ message TColumnShardConfig {
15531553

15541554
optional uint32 MaxInFlightIntervalsOnRequest = 16;
15551555
optional uint32 MaxReadStaleness_ms = 18 [default = 300000];
1556+
optional uint32 GCIntervalMs = 19 [default = 30000];
1557+
optional uint32 CompactionActualizationLagMs = 20 [default = 1000];
1558+
optional uint32 ActualizationTasksLagMs = 21 [default = 1000];
1559+
optional uint32 LagForCompactionBeforeTieringsMs = 22 [default = 3600000];
1560+
optional uint32 OptimizerFreshnessCheckDurationMs = 23 [default = 300000];
1561+
optional uint32 SmallPortionDetectSizeLimit = 24 [default = 1048576]; // 1 << 20
15561562
}
15571563

15581564
message TSchemeShardConfig {

‎ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ std::shared_ptr<NBlobOperations::NBlobStorage::TGCTask> TBlobManager::BuildGCTas
309309
return nullptr;
310310
}
311311

312-
if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod(TDuration::Seconds(GC_INTERVAL_SECONDS))) {
312+
if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod()) {
313313
ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep)("reason", "too_often");
314314
BlobsManagerCounters.GCCounters.SkipCollectionThrottling->Add(1);
315315
return nullptr;

‎ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,9 +133,6 @@ struct TBlobManagerCounters {
133133

134134
// The implementation of BlobManager that hides all GC-related details
135135
class TBlobManager : public IBlobManager, public TCommonBlobsTracker {
136-
private:
137-
static constexpr ui64 GC_INTERVAL_SECONDS = 30;
138-
139136
private:
140137
using TBlobAddress = NBlobOperations::NBlobStorage::TBlobAddress;
141138
class TGCContext;

0 commit comments

Comments
 (0)