Skip to content

Commit 80dd308

Browse files
ivanmorozov333ivanmorozov333
authored
clean useless scheme versions (#20476)
1 parent ea3edcf commit 80dd308

34 files changed

+1026
-149
lines changed

‎ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ class TInFlightComputes {
300300
sendTask->Info->IncPending();
301301
UndefinedShardTaskData.emplace_back(std::move(sendTask));
302302
} else {
303-
AFL_ENSURE (*computeShardId < ComputeActors.size())("compute_shard_id", *computeShardId);
303+
AFL_ENSURE(*computeShardId < ComputeActors.size())("compute_shard_id", *computeShardId);
304304
ComputeActors[*computeShardId].AddDataToSend(std::move(sendTask));
305305
}
306306
}
@@ -408,7 +408,7 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics {
408408
it->second->Stop(true, "");
409409
ShardScanners.erase(it);
410410
} else {
411-
AFL_VERIFY(!state->ActorId);
411+
AFL_ENSURE(!state->ActorId);
412412
}
413413

414414
if (stopShard) {

‎ydb/core/kqp/ut/olap/combinatory/compaction.cpp

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,60 @@
11
#include "compaction.h"
22

3+
#include <ydb/core/base/tablet_pipecache.h>
34
#include <ydb/core/tx/columnshard/hooks/abstract/abstract.h>
45
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
56

67
namespace NKikimr::NKqp {
78

9+
TConclusionStatus TRestartTabletsCommand::DoExecute(TKikimrRunner& kikimr) {
10+
auto csController = NYDBTest::TControllers::GetControllerAs<NYDBTest::NColumnShard::TController>();
11+
for (auto&& i : csController->GetShardActualIds()) {
12+
kikimr.GetTestServer().GetRuntime()->Send(
13+
MakePipePerNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward(new TEvents::TEvPoisonPill(), i, false));
14+
}
15+
return TConclusionStatus::Success();
16+
}
17+
18+
TConclusionStatus TStopSchemasCleanupCommand::DoExecute(TKikimrRunner& /*kikimr*/) {
19+
auto controller = NYDBTest::TControllers::GetControllerAs<NYDBTest::NColumnShard::TController>();
20+
AFL_VERIFY(controller);
21+
controller->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::CleanupSchemas);
22+
return TConclusionStatus::Success();
23+
}
24+
25+
TConclusionStatus TOneSchemasCleanupCommand::DoExecute(TKikimrRunner& /*kikimr*/) {
26+
auto controller = NYDBTest::TControllers::GetControllerAs<NYDBTest::NColumnShard::TController>();
27+
AFL_VERIFY(controller);
28+
AFL_VERIFY(!controller->IsBackgroundEnable(NKikimr::NYDBTest::ICSController::EBackground::CleanupSchemas));
29+
const i64 cleanups = controller->GetCleanupSchemasFinishedCounter().Val();
30+
controller->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::CleanupSchemas);
31+
const TInstant start = TInstant::Now();
32+
while (TInstant::Now() - start < TDuration::Seconds(10)) {
33+
if (cleanups < controller->GetCleanupSchemasFinishedCounter().Val()) {
34+
Cerr << "SCHEMAS_CLEANUP_HAPPENED: " << cleanups << " -> " << controller->GetCleanupSchemasFinishedCounter().Val() << Endl;
35+
break;
36+
}
37+
38+
Cerr << "WAIT_SCHEMAS_CLEANUP: " << controller->GetCleanupSchemasFinishedCounter().Val() << Endl;
39+
Sleep(TDuration::MilliSeconds(300));
40+
}
41+
42+
if (Expected) {
43+
AFL_VERIFY((cleanups < controller->GetCleanupSchemasFinishedCounter().Val()) == *Expected);
44+
}
45+
46+
controller->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::CleanupSchemas);
47+
return TConclusionStatus::Success();
48+
}
49+
50+
TConclusionStatus TFastPortionsCleanupCommand::DoExecute(TKikimrRunner& /*kikimr*/) {
51+
auto controller = NYDBTest::TControllers::GetControllerAs<NYDBTest::NColumnShard::TController>();
52+
AFL_VERIFY(controller);
53+
controller->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1));
54+
controller->SetOverrideMaxReadStaleness(TDuration::Seconds(1));
55+
return TConclusionStatus::Success();
56+
}
57+
858
TConclusionStatus TStopCompactionCommand::DoExecute(TKikimrRunner& /*kikimr*/) {
959
auto controller = NYDBTest::TControllers::GetControllerAs<NYDBTest::NColumnShard::TController>();
1060
AFL_VERIFY(controller);
@@ -30,7 +80,7 @@ TConclusionStatus TOneCompactionCommand::DoExecute(TKikimrRunner& /*kikimr*/) {
3080
}
3181

3282
AFL_VERIFY(compactions < controller->GetCompactionFinishedCounter().Val());
33-
83+
3484
controller->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction);
3585
return TConclusionStatus::Success();
3686
}

‎ydb/core/kqp/ut/olap/combinatory/compaction.h

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,61 @@
66

77
#include <ydb/library/conclusion/status.h>
88

9+
#include <util/string/type.h>
10+
911
namespace NKikimr::NKqp {
1012

13+
class TRestartTabletsCommand: public ICommand {
14+
private:
15+
virtual TConclusionStatus DoExecute(TKikimrRunner& /*kikimr*/) override;
16+
17+
public:
18+
TRestartTabletsCommand() {
19+
}
20+
};
21+
22+
class TStopSchemasCleanupCommand: public ICommand {
23+
private:
24+
virtual TConclusionStatus DoExecute(TKikimrRunner& /*kikimr*/) override;
25+
26+
public:
27+
TStopSchemasCleanupCommand() {
28+
}
29+
};
30+
31+
class TOneSchemasCleanupCommand: public ICommand {
32+
private:
33+
std::optional<bool> Expected;
34+
35+
virtual std::set<TString> DoGetCommandProperties() const override {
36+
return { "EXPECTED" };
37+
}
38+
virtual TConclusionStatus DoExecute(TKikimrRunner& /*kikimr*/) override;
39+
40+
virtual TConclusionStatus DoDeserializeProperties(const TPropertiesCollection& props) override {
41+
if (props.GetFreeArgumentsCount() != 0) {
42+
return TConclusionStatus::Fail("no free arguments have to been in one schemas cleanup command");
43+
}
44+
if (auto expected = props.GetOptional("EXPECTED")) {
45+
Expected = IsTrue(*expected);
46+
}
47+
return TConclusionStatus::Success();
48+
}
49+
50+
public:
51+
TOneSchemasCleanupCommand() {
52+
}
53+
};
54+
55+
class TFastPortionsCleanupCommand: public ICommand {
56+
private:
57+
virtual TConclusionStatus DoExecute(TKikimrRunner& /*kikimr*/) override;
58+
59+
public:
60+
TFastPortionsCleanupCommand() {
61+
}
62+
};
63+
1164
class TStopCompactionCommand: public ICommand {
1265
private:
1366
virtual TConclusionStatus DoExecute(TKikimrRunner& /*kikimr*/) override;

‎ydb/core/kqp/ut/olap/combinatory/select.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ TConclusionStatus TSelectCommand::DoExecute(TKikimrRunner& kikimr) {
2626
Cerr << "COMPARE: " << Compare << Endl;
2727
Cerr << "OUTPUT: " << output << Endl;
2828
CompareYson(output, Compare);
29+
} else {
30+
Cerr << "OUTPUT: " << output << Endl;
2931
}
3032
const ui32 iSkip = controller->GetIndexesSkippingOnSelect().Val() - indexSkipStart;
3133
const ui32 iNoData = controller->GetIndexesSkippedNoData().Val() - indexNoDataStart;

‎ydb/core/kqp/ut/olap/combinatory/variator.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ namespace {
1616
{"READ", []() { return std::make_shared<TSelectCommand>(); }},
1717
{"WAIT_COMPACTION", []() { return std::make_shared<TWaitCompactionCommand>(); }},
1818
{"STOP_COMPACTION", []() { return std::make_shared<TStopCompactionCommand>(); }},
19+
{"STOP_SCHEMAS_CLEANUP", []() { return std::make_shared<TStopSchemasCleanupCommand>(); }},
20+
{"ONE_SCHEMAS_CLEANUP", []() { return std::make_shared<TOneSchemasCleanupCommand>(); }},
21+
{"FAST_PORTIONS_CLEANUP", []() { return std::make_shared<TFastPortionsCleanupCommand>(); }},
1922
{"ONE_COMPACTION", []() { return std::make_shared<TOneCompactionCommand>(); }},
20-
{"ONE_ACTUALIZATION", []() { return std::make_shared<TOneActualizationCommand>(); }}
23+
{"ONE_ACTUALIZATION", []() { return std::make_shared<TOneActualizationCommand>(); }},
24+
{"RESTART_TABLETS", []() { return std::make_shared<TRestartTabletsCommand>(); }}
2125
};
2226

2327
std::pair<TString, TString> ParseCommandString(const TString& command) {

‎ydb/core/kqp/ut/olap/kqp_olap_ut.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2128,7 +2128,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
21282128
auto sender = runtime->AllocateEdgeActor();
21292129

21302130
InitRoot(server, sender);
2131-
Tests::NCommon::TLoggerInit(runtime).Initialize();
2131+
// Tests::NCommon::TLoggerInit(runtime).Initialize();
21322132

21332133
ui32 numShards = NSan::PlainOrUnderSanitizer(100, 10);
21342134
ui32 numIterations = NSan::PlainOrUnderSanitizer(100, 10);

‎ydb/core/kqp/ut/olap/tiering_ut.cpp

Lines changed: 32 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55

66
#include <ydb/core/kqp/ut/common/columnshard.h>
77
#include <ydb/core/tx/columnshard/data_locks/locks/list.h>
8+
#include <ydb/core/tx/columnshard/engines/changes/abstract/abstract.h>
89
#include <ydb/core/tx/columnshard/engines/scheme/abstract/index_info.h>
910
#include <ydb/core/tx/columnshard/hooks/testing/controller.h>
1011
#include <ydb/core/tx/columnshard/test_helper/controllers.h>
11-
#include <ydb/core/tx/columnshard/engines/changes/abstract/abstract.h>
1212
#include <ydb/core/wrappers/abstract.h>
1313
#include <ydb/core/wrappers/fake_storage.h>
1414

@@ -18,16 +18,14 @@ static const TString DEFAULT_TABLE_NAME = "/Root/olapStore/olapTable";
1818
static const TString DEFAULT_TIER_NAME = "/Root/tier1";
1919
static const TString DEFAULT_COLUMN_NAME = "timestamp";
2020

21-
class TAbortedWriteCounterController final
22-
: public NOlap::TWaitCompactionController {
21+
class TAbortedWriteCounterController final: public NOlap::TWaitCompactionController {
2322
public:
2423
ui64 GetAbortedWrites() const {
2524
return AbortedWrites.load();
2625
}
2726

2827
protected:
29-
bool DoOnWriteIndexComplete(const NOlap::TColumnEngineChanges& change,
30-
const ::NKikimr::NColumnShard::TColumnShard& shard) override {
28+
bool DoOnWriteIndexComplete(const NOlap::TColumnEngineChanges& change, const ::NKikimr::NColumnShard::TColumnShard& shard) override {
3129
if (change.IsAborted()) {
3230
++AbortedWrites;
3331
}
@@ -36,10 +34,10 @@ class TAbortedWriteCounterController final
3634
}
3735

3836
private:
39-
std::atomic<ui64> AbortedWrites{0};
37+
std::atomic<ui64> AbortedWrites{ 0 };
4038
};
4139

42-
template<class TCtrl = NOlap::TWaitCompactionController>
40+
template <class TCtrl = NOlap::TWaitCompactionController>
4341
class TTieringTestHelper {
4442
private:
4543
using TCtrlGuard = NYDBTest::TControllers::TGuard<TCtrl>;
@@ -87,14 +85,15 @@ class TTieringTestHelper {
8785
}
8886
}
8987

90-
void CheckAllDataInTier(const TString& tierName, const bool onlyActive=true) {
88+
void CheckAllDataInTier(const TString& tierName, const bool onlyActive = true) {
9189
NYdb::NTable::TTableClient tableClient = TestHelper->GetKikimr().GetTableClient();
9290

9391
auto selectQuery = TStringBuilder();
9492
selectQuery << R"(
9593
SELECT
9694
TierName, SUM(ColumnRawBytes) AS RawBytes, SUM(Rows) AS Rows
97-
FROM `)" << TablePath << R"(/.sys/primary_index_portion_stats`)";
95+
FROM `)" << TablePath
96+
<< R"(/.sys/primary_index_portion_stats`)";
9897
if (onlyActive) {
9998
selectQuery << " WHERE Activity == 1";
10099
}
@@ -107,7 +106,6 @@ class TTieringTestHelper {
107106
};
108107

109108
Y_UNIT_TEST_SUITE(KqpOlapTiering) {
110-
111109
Y_UNIT_TEST(EvictionResetTiering) {
112110
TTieringTestHelper tieringHelper;
113111
auto& csController = tieringHelper.GetCsController();
@@ -148,7 +146,8 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
148146
tieringHelper.CheckAllDataInTier(DEFAULT_TIER_NAME);
149147

150148
{
151-
const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P30000D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON timestamp)";
149+
const TString query =
150+
R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P30000D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON timestamp)";
152151
auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
153152
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
154153
}
@@ -203,13 +202,15 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
203202
testHelper.CreateTier("tier1");
204203

205204
{
206-
const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON unknown_column;)";
205+
const TString query =
206+
R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON unknown_column;)";
207207
auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
208208
UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
209209
}
210210

211211
{
212-
const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON uid;)";
212+
const TString query =
213+
R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("P10D") TO EXTERNAL DATA SOURCE `/Root/tier1` ON uid;)";
213214
auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
214215
UNIT_ASSERT_VALUES_UNEQUAL(result.GetStatus(), NYdb::EStatus::SUCCESS);
215216
}
@@ -280,19 +281,19 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
280281
{
281282
const TDuration tsInterval = TDuration::Days(3650);
282283
const ui64 rows = 10000;
283-
WriteTestData(testHelper.GetKikimr(), DEFAULT_TABLE_NAME, 0, (TInstant::Now() - tsInterval).MicroSeconds(), rows,
284-
false, tsInterval.MicroSeconds() / rows);
284+
WriteTestData(testHelper.GetKikimr(), DEFAULT_TABLE_NAME, 0, (TInstant::Now() - tsInterval).MicroSeconds(), rows, false,
285+
tsInterval.MicroSeconds() / rows);
285286
}
286287

287-
// {
288-
// auto selectQuery = TString(R"(
289-
// SELECT MAX(timestamp) AS timestamp FROM `/Root/olapStore/olapTable`
290-
// )");
291-
//
292-
// auto rows = ExecuteScanQuery(tableClient, selectQuery);
293-
// UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1);
294-
// UNIT_ASSERT_GT(GetTimestamp(rows[0].at(DEFAULT_COLUMN_NAME)), TInstant::Now() - TDuration::Days(100));
295-
// }
288+
// {
289+
// auto selectQuery = TString(R"(
290+
// SELECT MAX(timestamp) AS timestamp FROM `/Root/olapStore/olapTable`
291+
// )");
292+
//
293+
// auto rows = ExecuteScanQuery(tableClient, selectQuery);
294+
// UNIT_ASSERT_VALUES_EQUAL(rows.size(), 1);
295+
// UNIT_ASSERT_GT(GetTimestamp(rows[0].at(DEFAULT_COLUMN_NAME)), TInstant::Now() - TDuration::Days(100));
296+
// }
296297

297298
{
298299
auto selectQuery = TString(R"(
@@ -333,20 +334,20 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
333334
NYdb::NTable::TTableClient tableClient = kikimr.GetTableClient();
334335

335336
olapHelper.CreateTestOlapTable();
336-
const auto describeResult = kikimr.GetTestClient().Describe(
337-
kikimr.GetTestServer().GetRuntime(), "Root/olapStore/olapTable");
337+
const auto describeResult = kikimr.GetTestClient().Describe(kikimr.GetTestServer().GetRuntime(), "Root/olapStore/olapTable");
338338
const auto tablePathId = NColumnShard::TSchemeShardLocalPathId::FromRawValue(describeResult.GetPathId());
339339

340340
tieringHelper.WriteSampleData();
341341
csController->WaitCompactions(TDuration::Seconds(5));
342342
THashSet<NColumnShard::TInternalPathId> pathsToLock;
343-
for (const auto& [_, pathIdTranslator]: csController->GetActiveTablets()) {
343+
for (const auto& [_, pathIdTranslator] : csController->GetActiveTablets()) {
344344
if (auto internalPathId = pathIdTranslator->ResolveInternalPathId(tablePathId)) {
345345
pathsToLock.insert(*internalPathId);
346346
}
347347
};
348348

349-
csController->RegisterLock("table", std::make_shared<NOlap::NDataLocks::TListTablesLock>("table", std::move(pathsToLock), NOlap::NDataLocks::ELockCategory::Compaction));
349+
csController->RegisterLock("table",
350+
std::make_shared<NOlap::NDataLocks::TListTablesLock>("table", std::move(pathsToLock), NOlap::NDataLocks::ELockCategory::Compaction));
350351
{
351352
const TString query = R"(ALTER TABLE `/Root/olapStore/olapTable` SET TTL Interval("PT1S") ON timestamp)";
352353
auto result = testHelper.GetSession().ExecuteSchemeQuery(query).GetValueSync();
@@ -373,6 +374,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
373374
}
374375

375376
Y_UNIT_TEST(TieringGC) {
377+
Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->Clear();
376378
TTieringTestHelper tieringHelper;
377379
auto& csController = tieringHelper.GetCsController();
378380
csController->SetOverrideMaxReadStaleness(TDuration::Seconds(1));
@@ -383,7 +385,6 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
383385
olapHelper.CreateTestOlapTable();
384386
testHelper.CreateTier("tier1");
385387
tieringHelper.WriteSampleData();
386-
387388
testHelper.SetTiering(DEFAULT_TABLE_NAME, DEFAULT_TIER_NAME, DEFAULT_COLUMN_NAME);
388389
csController->WaitCompactions(TDuration::Seconds(5));
389390
csController->WaitActualization(TDuration::Seconds(5));
@@ -416,23 +417,19 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) {
416417
auto& olapHelper = tieringHelper.GetOlapHelper();
417418
auto& testHelper = tieringHelper.GetTestHelper();
418419

419-
420420
olapHelper.CreateTestOlapTable();
421421
testHelper.CreateTier("tier1");
422422
tieringHelper.WriteSampleData();
423423
putController->WaitCompactions(TDuration::Seconds(5));
424424

425425
putController->SetExternalStorageUnavailable(true);
426-
testHelper.SetTiering(DEFAULT_TABLE_NAME,
427-
DEFAULT_TIER_NAME,
428-
DEFAULT_COLUMN_NAME);
426+
testHelper.SetTiering(DEFAULT_TABLE_NAME, DEFAULT_TIER_NAME, DEFAULT_COLUMN_NAME);
429427

430428
putController->WaitActualization(TDuration::Seconds(5));
431429
Sleep(TDuration::Seconds(5));
432430

433431
UNIT_ASSERT_C(putController->GetAbortedWrites() > 20,
434-
"Expected load spike, but only "
435-
<< putController->GetAbortedWrites() << " PutObject requests recorded"); // comment after fix
432+
"Expected load spike, but only " << putController->GetAbortedWrites() << " PutObject requests recorded"); // comment after fix
436433
// UNIT_ASSERT_C(putController->GetAbortedWrites() < 10,
437434
// "Expected load spike, but was "
438435
// << putController->GetAbortedWrites() << " PutObject requests recorded"); // uncomment after fix

0 commit comments

Comments
 (0)