From 05ca37fe3913534407351352494709256c065120 Mon Sep 17 00:00:00 2001 From: Semyon Date: Thu, 29 May 2025 01:17:46 +0300 Subject: [PATCH 01/62] application of duplicate filters in simple reader (#18608) --- ydb/core/formats/arrow/arrow_filter.cpp | 57 ++++++++++++++++++- ydb/core/formats/arrow/arrow_filter.h | 5 +- .../formats/arrow/ut/ut_column_filter.cpp | 21 +++++++ .../columnshard/columnshard_private_events.h | 2 + ydb/core/tx/columnshard/counters/scan.h | 11 +++- .../engines/reader/abstract/read_metadata.h | 5 ++ .../engines/reader/common/description.h | 6 ++ .../constructor/read_metadata.cpp | 1 + .../simple_reader/duplicates/events.cpp | 16 ++++++ .../reader/simple_reader/duplicates/events.h | 35 ++++++++++++ .../simple_reader/duplicates/manager.cpp | 17 ++++++ .../reader/simple_reader/duplicates/manager.h | 33 +++++++++++ .../reader/simple_reader/duplicates/ya.make | 12 ++++ .../reader/simple_reader/iterator/context.cpp | 29 +++++++--- .../reader/simple_reader/iterator/context.h | 26 +++++++-- .../simple_reader/iterator/fetching.cpp | 34 +++++++++++ .../reader/simple_reader/iterator/fetching.h | 25 ++++++++ .../simple_reader/iterator/plain_read_data.h | 2 + .../reader/simple_reader/iterator/source.h | 19 ++++++- .../reader/simple_reader/iterator/ya.make | 1 + .../reader/transaction/tx_internal_scan.cpp | 1 + 21 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.cpp create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make diff --git a/ydb/core/formats/arrow/arrow_filter.cpp b/ydb/core/formats/arrow/arrow_filter.cpp index 7be4883bea85..f7544701c2ca 100644 --- a/ydb/core/formats/arrow/arrow_filter.cpp +++ b/ydb/core/formats/arrow/arrow_filter.cpp @@ -297,6 +297,9 @@ const std::vector& TColumnFilter::BuildSimpleFilter() const { class TMergePolicyAnd { private: public: + static bool HasValue(const bool /*a*/, const bool /*b*/) { + return true; + } static bool Calc(const bool a, const bool b) { return a && b; } @@ -307,11 +310,17 @@ class TMergePolicyAnd { return TColumnFilter::BuildDenyFilter(); } } + static TColumnFilter MergeWithSimple(const bool simpleValue, const TColumnFilter& filter) { + return MergeWithSimple(filter, simpleValue); + } }; class TMergePolicyOr { private: public: + static bool HasValue(const bool /*a*/, const bool /*b*/) { + return true; + } static bool Calc(const bool a, const bool b) { return a || b; } @@ -322,6 +331,43 @@ class TMergePolicyOr { return filter; } } + static TColumnFilter MergeWithSimple(const bool simpleValue, const TColumnFilter& filter) { + return MergeWithSimple(filter, simpleValue); + } +}; + +class TMergePolicyApplyFilter { +private: +public: + static bool HasValue(const bool /*a*/, const bool b) { + return b; + } + static bool Calc(const bool a, const bool /*b*/) { + return a; + } + static TColumnFilter MergeWithSimple(const TColumnFilter& filter, const bool simpleValue) { + if (simpleValue) { + return filter; + } else { + return TColumnFilter::BuildDenyFilter(); + } + } + static TColumnFilter MergeWithSimple(const bool simpleValue, const TColumnFilter& filter) { + const auto count = filter.GetRecordsCount(); + if (simpleValue) { + TColumnFilter result = TColumnFilter::BuildAllowFilter(); + if (count) { + result.Add(true, *count); + } + return result; + } else { + TColumnFilter result = TColumnFilter::BuildDenyFilter(); + if (count) { + result.Add(false, *count); + } + return result; + } + } }; class TColumnFilter::TMergerImpl { @@ -340,7 +386,7 @@ class TColumnFilter::TMergerImpl { if (Filter1.empty() && Filter2.empty()) { return TColumnFilter(TMergePolicy::Calc(Filter1.DefaultFilterValue, Filter2.DefaultFilterValue)); } else if (Filter1.empty()) { - return TMergePolicy::MergeWithSimple(Filter2, Filter1.DefaultFilterValue); + return TMergePolicy::MergeWithSimple(Filter1.DefaultFilterValue, Filter2); } else if (Filter2.empty()) { return TMergePolicy::MergeWithSimple(Filter1, Filter2.DefaultFilterValue); } else { @@ -359,7 +405,7 @@ class TColumnFilter::TMergerImpl { while (it1 != Filter1.Filter.cend() && it2 != Filter2.Filter.cend()) { const ui32 delta = TColumnFilter::CrossSize(pos2, pos2 + *it2, pos1, pos1 + *it1); - if (delta) { + if (delta && TMergePolicy::HasValue(curValue1, curValue2)) { if (!count || curCurrent != TMergePolicy::Calc(curValue1, curValue2)) { resultFilter.emplace_back(delta); curCurrent = TMergePolicy::Calc(curValue1, curValue2); @@ -405,10 +451,15 @@ TColumnFilter TColumnFilter::Or(const TColumnFilter& extFilter) const { return TMergerImpl(*this, extFilter).Merge(); } +TColumnFilter TColumnFilter::ApplyFilterFrom(const TColumnFilter& extFilter) const { + ResetCaches(); + return TMergerImpl(*this, extFilter).Merge(); +} + TColumnFilter TColumnFilter::CombineSequentialAnd(const TColumnFilter& extFilter) const { ResetCaches(); if (Filter.empty()) { - return TMergePolicyAnd::MergeWithSimple(extFilter, DefaultFilterValue); + return TMergePolicyAnd::MergeWithSimple(DefaultFilterValue, extFilter); } else if (extFilter.Filter.empty()) { return TMergePolicyAnd::MergeWithSimple(*this, extFilter.DefaultFilterValue); } else { diff --git a/ydb/core/formats/arrow/arrow_filter.h b/ydb/core/formats/arrow/arrow_filter.h index 9a135f5961ae..4fae08842234 100644 --- a/ydb/core/formats/arrow/arrow_filter.h +++ b/ydb/core/formats/arrow/arrow_filter.h @@ -263,8 +263,9 @@ class TColumnFilter { return TColumnFilter(false); } - TColumnFilter And(const TColumnFilter& extFilter) const Y_WARN_UNUSED_RESULT; - TColumnFilter Or(const TColumnFilter& extFilter) const Y_WARN_UNUSED_RESULT; + [[nodiscard]] TColumnFilter And(const TColumnFilter& extFilter) const; + [[nodiscard]] TColumnFilter Or(const TColumnFilter& extFilter) const; + [[nodiscard]] TColumnFilter ApplyFilterFrom(const TColumnFilter& filter) const; class TApplyContext { private: diff --git a/ydb/core/formats/arrow/ut/ut_column_filter.cpp b/ydb/core/formats/arrow/ut/ut_column_filter.cpp index 5994e8c45ce6..11acbe162613 100644 --- a/ydb/core/formats/arrow/ut/ut_column_filter.cpp +++ b/ydb/core/formats/arrow/ut/ut_column_filter.cpp @@ -28,6 +28,27 @@ Y_UNIT_TEST_SUITE(ColumnFilter) { UNIT_ASSERT_VALUES_EQUAL(JoinSeq(",", resultVec), "1,1,1,0,1,0,0"); } + Y_UNIT_TEST(ApplyFilterToFilter) { + TColumnFilter filter1({ true, true, true, false, true, true, false }); + TColumnFilter filter2({ false, true, true, true, true, false, false }); + + { + auto result = filter1.ApplyFilterFrom(filter2); + + UNIT_ASSERT_VALUES_EQUAL(result.GetRecordsCountVerified(), 4); + auto resultVec = result.BuildSimpleFilter(); + UNIT_ASSERT_VALUES_EQUAL(JoinSeq(",", resultVec), "1,1,0,1"); + } + + { + auto result = filter2.ApplyFilterFrom(filter1); + + UNIT_ASSERT_VALUES_EQUAL(result.GetRecordsCountVerified(), 5); + auto resultVec = result.BuildSimpleFilter(); + UNIT_ASSERT_VALUES_EQUAL(JoinSeq(",", resultVec), "0,1,1,1,0"); + } + } + Y_UNIT_TEST(FilterSlice) { TColumnFilter filter({ true, true, true, false, true, true, false }); { diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index c5c4d492efd5..4d711600224c 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -70,6 +70,8 @@ struct TEvPrivate { EvRemovePortionDataAccessor, EvMetadataAccessorsInfo, + EvRequestFilter, + EvEnd }; diff --git a/ydb/core/tx/columnshard/counters/scan.h b/ydb/core/tx/columnshard/counters/scan.h index aa7db8a8adfc..897610fbbdba 100644 --- a/ydb/core/tx/columnshard/counters/scan.h +++ b/ydb/core/tx/columnshard/counters/scan.h @@ -360,6 +360,7 @@ class TConcreteScanCounters: public TScanCounters { std::shared_ptr ResourcesAllocationTasksCount = std::make_shared(); std::shared_ptr ResultsForSourceCount = std::make_shared(); std::shared_ptr ResultsForReplyGuard = std::make_shared(); + std::shared_ptr FilterFetchingGuard = std::make_shared(); std::shared_ptr TotalExecutionDurationUs = std::make_shared(); THashMap> SkipNodesCount; THashMap> ExecuteNodesCount; @@ -399,7 +400,8 @@ class TConcreteScanCounters: public TScanCounters { << "ReadTasksCount:" << ReadTasksCount->Val() << ";" << "ResourcesAllocationTasksCount:" << ResourcesAllocationTasksCount->Val() << ";" << "ResultsForSourceCount:" << ResultsForSourceCount->Val() << ";" - << "ResultsForReplyGuard:" << ResultsForReplyGuard->Val() << ";"; + << "ResultsForReplyGuard:" << ResultsForReplyGuard->Val() << ";" + << "FilterFetchingGuard:" << FilterFetchingGuard->Val() << ";"; } TCounterGuard GetResultsForReplyGuard() const { @@ -434,9 +436,14 @@ class TConcreteScanCounters: public TScanCounters { return TCounterGuard(AssembleTasksCount); } + TCounterGuard GetFilterFetchingGuard() const { + return TCounterGuard(FilterFetchingGuard); + } + bool InWaiting() const { return MergeTasksCount->Val() || AssembleTasksCount->Val() || ReadTasksCount->Val() || ResourcesAllocationTasksCount->Val() || - FetchAccessorsCount->Val() || ResultsForSourceCount->Val() || FetchBlobsCount->Val() || ResultsForReplyGuard->Val(); + FetchAccessorsCount->Val() || ResultsForSourceCount->Val() || FetchBlobsCount->Val() || ResultsForReplyGuard->Val() || + FilterFetchingGuard->Val(); } const THashMap>& GetSkipStats() const { diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h index aac729f0a4aa..b369cec47373 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h @@ -58,6 +58,7 @@ class TReadMetadataBase { std::shared_ptr ResultIndexSchema; ui64 TxId = 0; std::optional LockId; + EDeduplicationPolicy DeduplicationPolicy = EDeduplicationPolicy::ALLOW_DUPLICATES; public: using TConstPtr = std::shared_ptr; @@ -107,6 +108,10 @@ class TReadMetadataBase { return LockId; } + EDeduplicationPolicy GetDeduplicationPolicy() const { + return DeduplicationPolicy; + } + void OnReadFinished(NColumnShard::TColumnShard& owner) const { DoOnReadFinished(owner); } diff --git a/ydb/core/tx/columnshard/engines/reader/common/description.h b/ydb/core/tx/columnshard/engines/reader/common/description.h index 0572c4e8fc40..8c1edddea79d 100644 --- a/ydb/core/tx/columnshard/engines/reader/common/description.h +++ b/ydb/core/tx/columnshard/engines/reader/common/description.h @@ -13,6 +13,11 @@ enum class ERequestSorting { DESC /* "descending" */, }; +enum class EDeduplicationPolicy { + ALLOW_DUPLICATES = 0, + PREVENT_DUPLICATES, +}; + // Describes read/scan request struct TReadDescription { private: @@ -35,6 +40,7 @@ struct TReadDescription { // operations with potentially different columns. We have to remove columns to support -Inf (Null) and +Inf. std::shared_ptr PKRangesFilter; NYql::NDqProto::EDqStatsMode StatsMode = NYql::NDqProto::EDqStatsMode::DQ_STATS_MODE_NONE; + EDeduplicationPolicy DeduplicationPolicy = EDeduplicationPolicy::ALLOW_DUPLICATES; // List of columns std::vector ColumnIds; diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp index 1e3999e2ede7..934a603e4df3 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp @@ -41,6 +41,7 @@ TConclusionStatus TReadMetadata::Init( } StatsMode = readDescription.StatsMode; + DeduplicationPolicy = readDescription.DeduplicationPolicy; return TConclusionStatus::Success(); } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.cpp new file mode 100644 index 000000000000..602e4aa26bb5 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.cpp @@ -0,0 +1,16 @@ +#include "events.h" + +#include + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +TEvRequestFilter::TEvRequestFilter(const IDataSource& source, const std::shared_ptr& subscriber) + : MinPK(source.GetMinPK()) + , MaxPK(source.GetMaxPK()) + , SourceId(source.GetSourceId()) + , RecordsCount(source.GetRecordsCount()) + , MaxVersion(source.GetContext()->GetCommonContext()->GetReadMetadata()->GetRequestSnapshot()) + , Subscriber(subscriber) { +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h new file mode 100644 index 000000000000..6a386fdde4f0 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NReader::NSimple { +class IDataSource; +} + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +class IFilterSubscriber { +public: + virtual void OnFilterReady(NArrow::TColumnFilter&&) = 0; + virtual void OnFailure(const TString& reason) = 0; + virtual ~IFilterSubscriber() = default; +}; + +class TEvRequestFilter: public NActors::TEventLocal { +private: + NArrow::TSimpleRow MinPK; + NArrow::TSimpleRow MaxPK; + YDB_READONLY_DEF(ui64, SourceId); + YDB_READONLY_DEF(ui64, RecordsCount); + TSnapshot MaxVersion; + YDB_READONLY_DEF(std::shared_ptr, Subscriber); + +public: + TEvRequestFilter(const IDataSource& source, const std::shared_ptr& subscriber); +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp new file mode 100644 index 000000000000..6a856b5b8025 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.cpp @@ -0,0 +1,17 @@ +#include "manager.h" + +#include + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +TDuplicateManager::TDuplicateManager(const TSpecialReadContext& /*context*/) + : TActor(&TDuplicateManager::StateMain) { +} + +void TDuplicateManager::Handle(const TEvRequestFilter::TPtr& ev) { + NArrow::TColumnFilter filter = NArrow::TColumnFilter::BuildAllowFilter(); + filter.Add(true, ev->Get()->GetRecordsCount()); + ev->Get()->GetSubscriber()->OnFilterReady(std::move(filter)); +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h new file mode 100644 index 000000000000..9b710f359259 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/manager.h @@ -0,0 +1,33 @@ +#pragma once + +#include "events.h" + +#include + +namespace NKikimr::NOlap::NReader::NSimple { +class TSpecialReadContext; +} + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +class TDuplicateManager: public NActors::TActor { +private: + STATEFN(StateMain) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvRequestFilter, Handle); + hFunc(NActors::TEvents::TEvPoison, Handle); + default: + AFL_VERIFY(false)("unexpected_event", ev->GetTypeName()); + } + } + + void Handle(const TEvRequestFilter::TPtr&); + void Handle(const NActors::TEvents::TEvPoison::TPtr&) { + PassAway(); + } + +public: + TDuplicateManager(const TSpecialReadContext& context); +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make new file mode 100644 index 000000000000..3e9720055408 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( + manager.cpp + events.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/engines/reader/common_reader/iterator +) + +END() diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp index 59f3601bf55e..59c560c9e982 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp @@ -2,6 +2,7 @@ #include "source.h" #include +#include #include namespace NKikimr::NOlap::NReader::NSimple { @@ -42,14 +43,15 @@ std::shared_ptr TSpecialReadContext::DoGetColumnsFetchingPlan(c needShardingFilter = true; } } + const bool preventDuplicates = GetReadMetadata()->GetDeduplicationPolicy() == EDeduplicationPolicy::PREVENT_DUPLICATES; { auto& result = CacheFetchingScripts[needSnapshots ? 1 : 0][partialUsageByPK ? 1 : 0][useIndexes ? 1 : 0][needShardingFilter ? 1 : 0] - [hasDeletions ? 1 : 0]; + [hasDeletions ? 1 : 0][preventDuplicates ? 1 : 0]; if (result.NeedInitialization()) { TGuard g(Mutex); if (auto gInit = result.StartInitialization()) { gInit->InitializationFinished( - BuildColumnsFetchingPlan(needSnapshots, partialUsageByPK, useIndexes, needShardingFilter, hasDeletions)); + BuildColumnsFetchingPlan(needSnapshots, partialUsageByPK, useIndexes, needShardingFilter, hasDeletions, preventDuplicates)); } AFL_VERIFY(!result.NeedInitialization()); } @@ -58,7 +60,7 @@ std::shared_ptr TSpecialReadContext::DoGetColumnsFetchingPlan(c } std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(const bool needSnapshots, const bool partialUsageByPredicateExt, - const bool /*useIndexes*/, const bool needFilterSharding, const bool needFilterDeletion) const { + const bool /*useIndexes*/, const bool needFilterSharding, const bool needFilterDeletion, const bool preventDuplicates) const { const bool partialUsageByPredicate = partialUsageByPredicateExt && GetPredicateColumns()->GetColumnsCount(); NCommon::TFetchingScriptBuilder acc(*this); @@ -91,6 +93,9 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c acc.AddAssembleStep(*GetSpecColumns(), "SPEC", NArrow::NSSA::IMemoryCalculationPolicy::EStage::Filter, false); acc.AddStep(std::make_shared()); } + if (preventDuplicates) { + acc.AddStep(std::make_shared()); + } const auto& chainProgram = GetReadMetadata()->GetProgram().GetChainVerified(); acc.AddStep(std::make_shared(chainProgram)); } @@ -99,8 +104,18 @@ std::shared_ptr TSpecialReadContext::BuildColumnsFetchingPlan(c return std::move(acc).Build(); } -TSpecialReadContext::TSpecialReadContext(const std::shared_ptr& commonContext) - : TBase(commonContext) { +void TSpecialReadContext::RegisterActors() { + AFL_VERIFY(!DuplicatesManager); + if (GetReadMetadata()->GetDeduplicationPolicy() == EDeduplicationPolicy::PREVENT_DUPLICATES) { + DuplicatesManager = NActors::TActivationContext::Register(new NDuplicateFiltering::TDuplicateManager(*this)); + } +} + +void TSpecialReadContext::UnregisterActors() { + if (DuplicatesManager) { + NActors::TActivationContext::AsActorContext().Send(DuplicatesManager, new NActors::TEvents::TEvPoison); + DuplicatesManager = TActorId(); + } } TString TSpecialReadContext::ProfileDebugString() const { @@ -109,8 +124,8 @@ TString TSpecialReadContext::ProfileDebugString() const { return (val & (1 << pos)) ? 1 : 0; }; - for (ui32 i = 0; i < (1 << 5); ++i) { - auto& script = CacheFetchingScripts[GetBit(i, 0)][GetBit(i, 1)][GetBit(i, 2)][GetBit(i, 3)][GetBit(i, 4)]; + for (ui32 i = 0; i < (1 << 6); ++i) { + auto& script = CacheFetchingScripts[GetBit(i, 0)][GetBit(i, 1)][GetBit(i, 2)][GetBit(i, 3)][GetBit(i, 4)][GetBit(i, 5)]; if (script.HasScript()) { sb << script.ProfileDebugString() << ";"; } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h index c1859a3062d7..cb9b07b98347 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.h @@ -11,18 +11,22 @@ namespace NKikimr::NOlap::NReader::NSimple { class IDataSource; +class TSourceConstructor; using TColumnsSet = NCommon::TColumnsSet; using TColumnsSetIds = NCommon::TColumnsSetIds; using EMemType = NCommon::EMemType; using TFetchingScript = NCommon::TFetchingScript; -class TSpecialReadContext: public NCommon::TSpecialReadContext { +class TSpecialReadContext: public NCommon::TSpecialReadContext, TNonCopyable { private: using TBase = NCommon::TSpecialReadContext; + TActorId DuplicatesManager = TActorId(); + +private: std::shared_ptr BuildColumnsFetchingPlan(const bool needSnapshots, const bool partialUsageByPredicateExt, - const bool useIndexes, const bool needFilterSharding, const bool needFilterDeletion) const; + const bool useIndexes, const bool needFilterSharding, const bool needFilterDeletion, const bool needFilterDuplicates) const; TMutex Mutex; - std::array, 2>, 2>, 2>, 2> CacheFetchingScripts; + std::array, 2>, 2>, 2>, 2>, 2> CacheFetchingScripts; std::shared_ptr AskAccumulatorsScript; virtual std::shared_ptr DoGetColumnsFetchingPlan(const std::shared_ptr& source) override; @@ -30,7 +34,21 @@ class TSpecialReadContext: public NCommon::TSpecialReadContext { public: virtual TString ProfileDebugString() const override; - TSpecialReadContext(const std::shared_ptr& commonContext); + void RegisterActors(); + void UnregisterActors(); + + const TActorId& GetDuplicatesManagerVerified() const { + AFL_VERIFY(DuplicatesManager); + return DuplicatesManager; + } + + TSpecialReadContext(const std::shared_ptr& commonContext) + : TBase(commonContext) { + } + + ~TSpecialReadContext() { + AFL_VERIFY(!DuplicatesManager); + } }; } // namespace NKikimr::NOlap::NReader::NSimple diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp index 687477154f52..1728fea5f647 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp @@ -3,6 +3,7 @@ #include "source.h" #include +#include #include #include @@ -192,4 +193,37 @@ TConclusion TPrepareResultStep::DoExecuteInplace(const std::shared_ptrGetSourceId())("filter", filter.DebugString()); + if (source->GetContext()->IsAborted()) { + return; + } + if (const std::shared_ptr appliedFilter = source->GetStageData().GetAppliedFilter()) { + filter = filter.ApplyFilterFrom(*appliedFilter); + } + source->MutableStageData().AddFilter(std::move(filter)); + Step.Next(); + auto task = std::make_shared(source, std::move(Step), source->GetContext()->GetCommonContext()->GetScanActorId(), false); + NConveyor::TScanServiceOperator::SendTaskToExecute(task, source->GetContext()->GetCommonContext()->GetConveyorProcessId()); + } +} + +void TDuplicateFilter::TFilterSubscriber::OnFailure(const TString& reason) { + if (auto source = Source.lock()) { + source->GetContext()->GetCommonContext()->AbortWithError("cannot build duplicate filter: " + reason); + } +} + +TDuplicateFilter::TFilterSubscriber::TFilterSubscriber(const std::shared_ptr& source, const TFetchingScriptCursor& step) + : Source(source) + , Step(step) + , TaskGuard(source->GetContext()->GetCommonContext()->GetCounters().GetFilterFetchingGuard()) { +} + +TConclusion TDuplicateFilter::DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const { + source->StartFetchingDuplicateFilter(std::make_shared(source, step)); + return false; +} + } // namespace NKikimr::NOlap::NReader::NSimple diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h index d1dcc822b035..3b9da2b28e01 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -212,4 +213,28 @@ class TShardingFilter: public IFetchingStep { } }; +class TDuplicateFilter: public IFetchingStep { +private: + using TBase = IFetchingStep; + + class TFilterSubscriber: public NDuplicateFiltering::IFilterSubscriber { + private: + std::weak_ptr Source; + TFetchingScriptCursor Step; + NColumnShard::TCounterGuard TaskGuard; + + virtual void OnFilterReady(NArrow::TColumnFilter&& filter) override; + virtual void OnFailure(const TString& reason) override; + + public: + TFilterSubscriber(const std::shared_ptr& source, const TFetchingScriptCursor& step); + }; + +public: + virtual TConclusion DoExecuteInplace(const std::shared_ptr& source, const TFetchingScriptCursor& step) const override; + TDuplicateFilter() + : TBase("DUPLICATE") { + } +}; + } // namespace NKikimr::NOlap::NReader::NSimple diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.h index 19c8e14acedf..5a64e9250e90 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.h @@ -19,6 +19,7 @@ class TPlainReadData: public IDataReader, TNonCopyable, NColumnShard::TMonitorin protected: virtual TConclusionStatus DoStart() override { + SpecialReadContext->RegisterActors(); return Scanner->Start(); } @@ -69,6 +70,7 @@ class TPlainReadData: public IDataReader, TNonCopyable, NColumnShard::TMonitorin if (SpecialReadContext->IsActive()) { Abort("unexpected on destructor"); } + SpecialReadContext->UnregisterActors(); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h index 0146875d97a5..d2a0d4564339 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h @@ -192,7 +192,7 @@ class IDataSource: public NCommon::IDataSource { const TReplaceKeyAdapter& GetStart() const { return Start; } - const TReplaceKeyAdapter GetFinish() const { + const TReplaceKeyAdapter& GetFinish() const { return Finish; } @@ -250,6 +250,12 @@ class IDataSource: public NCommon::IDataSource { return DoStartFetchingAccessor(sourcePtr, step); } + void StartFetchingDuplicateFilter(std::shared_ptr&& subscriber) { + NActors::TActivationContext::AsActorContext().Send( + std::static_pointer_cast(GetContext())->GetDuplicatesManagerVerified(), + new NDuplicateFiltering::TEvRequestFilter(*this, std::move(subscriber))); + } + virtual TInternalPathId GetPathId() const = 0; virtual bool HasIndexes(const std::set& indexIds) const = 0; @@ -260,6 +266,9 @@ class IDataSource: public NCommon::IDataSource { virtual ui64 GetIndexRawBytes(const std::set& indexIds) const = 0; + virtual NArrow::TSimpleRow GetMinPK() const = 0; + virtual NArrow::TSimpleRow GetMaxPK() const = 0; + void Abort() { DoAbort(); } @@ -430,6 +439,14 @@ class TPortionDataSource: public IDataSource { return GetStageData().GetPortionAccessor().GetIndexRawBytes(indexIds, false); } + virtual NArrow::TSimpleRow GetMinPK() const override { + return Portion->GetMeta().IndexKeyStart(); + } + + virtual NArrow::TSimpleRow GetMaxPK() const override { + return Portion->GetMeta().IndexKeyEnd(); + } + const TPortionInfo& GetPortionInfo() const { return *Portion; } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/ya.make index baff60941f27..37599539e477 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/ya.make +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/ya.make @@ -16,6 +16,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/reader/common_reader/iterator ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sync_points + ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates ydb/core/tx/conveyor/usage ydb/core/tx/limiter/grouped_memory/usage ) diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp index 14c0cb9ef088..863407ef3262 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp @@ -49,6 +49,7 @@ void TTxInternalScan::Complete(const TActorContext& ctx) { read.PathId = request.GetPathId(); read.LockId = LockId; read.ReadNothing = !Self->TablesManager.HasTable(read.PathId); + read.DeduplicationPolicy = EDeduplicationPolicy::PREVENT_DUPLICATES; std::unique_ptr scannerConstructor(new NPlain::TIndexScannerConstructor(context)); read.ColumnIds = request.GetColumnIds(); if (request.RangesFilter) { From 3cb66630f92c8a0c6c752ff8b7ccae7a3f303bfd Mon Sep 17 00:00:00 2001 From: Semyon Date: Tue, 3 Jun 2025 18:58:22 +0300 Subject: [PATCH 02/62] duplicate filter construction task (#19120) --- .../formats/arrow/reader/batch_iterator.h | 43 ++++++++--- ydb/core/formats/arrow/reader/merger.h | 12 +-- ydb/core/formats/arrow/ut/ut_arrow.cpp | 10 +-- .../columnshard/columnshard_private_events.h | 1 + ydb/core/tx/columnshard/common/snapshot.h | 4 + .../columnshard/counters/counters_manager.h | 2 + .../counters/duplicate_filtering.cpp | 10 +++ .../counters/duplicate_filtering.h | 26 +++++++ ydb/core/tx/columnshard/counters/ya.make | 1 + .../engines/changes/compaction/merger.cpp | 2 +- .../reader/plain_reader/iterator/merge.cpp | 3 +- .../simple_reader/duplicates/common.cpp | 3 + .../reader/simple_reader/duplicates/common.h | 61 +++++++++++++++ .../reader/simple_reader/duplicates/events.h | 30 ++++++++ .../reader/simple_reader/duplicates/merge.cpp | 76 +++++++++++++++++++ .../reader/simple_reader/duplicates/merge.h | 56 ++++++++++++++ .../reader/simple_reader/duplicates/ya.make | 2 + .../operations/slice_builder/pack_builder.cpp | 2 +- 18 files changed, 322 insertions(+), 22 deletions(-) create mode 100644 ydb/core/tx/columnshard/counters/duplicate_filtering.cpp create mode 100644 ydb/core/tx/columnshard/counters/duplicate_filtering.h create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.cpp create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.h create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp create mode 100644 ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.h diff --git a/ydb/core/formats/arrow/reader/batch_iterator.h b/ydb/core/formats/arrow/reader/batch_iterator.h index 91d47d78041c..5fba3f53fc50 100644 --- a/ydb/core/formats/arrow/reader/batch_iterator.h +++ b/ydb/core/formats/arrow/reader/batch_iterator.h @@ -4,6 +4,26 @@ namespace NKikimr::NArrow::NMerger { +class TIterationOrder { +private: + YDB_READONLY_DEF(bool, IsReversed); + YDB_READONLY_DEF(ui64, Start); + +public: + TIterationOrder(const bool reverse, const ui64 start) + : IsReversed(reverse) + , Start(start) { + } + + static TIterationOrder Forward(const ui64 start) { + return TIterationOrder(false, start); + } + + static TIterationOrder Reversed(const ui64 start) { + return TIterationOrder(true, start); + } +}; + class TBatchIterator { private: bool ControlPointFlag; @@ -16,11 +36,12 @@ class TBatchIterator { std::shared_ptr Filter; std::shared_ptr FilterIterator; - i32 GetFirstPosition() const { + i32 GetPosition(const ui64 position) const { + AFL_VERIFY((i64)position < RecordsCount); if (ReverseSortKff > 0) { - return 0; + return position; } else { - return RecordsCount - 1; + return RecordsCount - position - 1; } } @@ -52,20 +73,24 @@ class TBatchIterator { template TBatchIterator(std::shared_ptr batch, std::shared_ptr filter, const arrow::Schema& keySchema, - const arrow::Schema& dataSchema, const bool reverseSort, const std::vector& versionColumnNames, const ui64 sourceId) + const arrow::Schema& dataSchema, const std::vector& versionColumnNames, const ui64 sourceId, + const TIterationOrder& order = TIterationOrder::Forward(0)) : ControlPointFlag(false) - , KeyColumns(batch, 0, keySchema.field_names(), dataSchema.field_names(), reverseSort) + , KeyColumns(batch, 0, keySchema.field_names(), dataSchema.field_names(), order.GetIsReversed()) , VersionColumns(batch, 0, versionColumnNames, {}, false) , RecordsCount(batch->num_rows()) - , ReverseSortKff(reverseSort ? -1 : 1) + , ReverseSortKff(order.GetIsReversed() ? -1 : 1) , SourceId(sourceId) , Filter(filter) { AFL_VERIFY(KeyColumns.IsSameSortingSchema(keySchema))("batch", KeyColumns.DebugJson())("schema", keySchema.ToString()); AFL_VERIFY(KeyColumns.IsSameDataSchema(dataSchema))("batch", KeyColumns.DebugJson())("schema", dataSchema.ToString()); - Y_ABORT_UNLESS(KeyColumns.InitPosition(GetFirstPosition())); - Y_ABORT_UNLESS(VersionColumns.InitPosition(GetFirstPosition())); + Y_ABORT_UNLESS(KeyColumns.InitPosition(GetPosition(order.GetStart()))); + Y_ABORT_UNLESS(VersionColumns.InitPosition(GetPosition(order.GetStart()))); if (Filter) { - FilterIterator = std::make_shared(Filter->GetIterator(reverseSort, RecordsCount)); + FilterIterator = std::make_shared(Filter->GetIterator(order.GetIsReversed(), RecordsCount)); + if (order.GetStart()) { + FilterIterator->Next(order.GetStart()); + } } } diff --git a/ydb/core/formats/arrow/reader/merger.h b/ydb/core/formats/arrow/reader/merger.h index b49e1b6a67e1..282312361b28 100644 --- a/ydb/core/formats/arrow/reader/merger.h +++ b/ydb/core/formats/arrow/reader/merger.h @@ -159,18 +159,20 @@ class TMergePartialStream { } template - void AddSource(const std::shared_ptr& batch, const std::shared_ptr& filter, - const std::optional sourceIdExt = std::nullopt) { + void AddSource(const std::shared_ptr& batch, + const std::shared_ptr& filter, const TIterationOrder& order, const std::optional sourceIdExt = std::nullopt) { + AFL_VERIFY(order.GetIsReversed() == Reverse); const ui64 sourceId = sourceIdExt.value_or(SortHeap.Size()); - if (!batch || !batch->num_rows()) { + if (!batch || (i64)batch->num_rows() == (i64)order.GetStart()) { return; } + AFL_VERIFY((i64)order.GetStart() < (i64)batch->num_rows())("start", order.GetStart())("num_rows", batch->num_rows()); // Y_DEBUG_ABORT_UNLESS(NArrow::IsSorted(batch, SortSchema)); const bool isDenyFilter = filter && filter->IsTotalDenyFilter(); auto filterImpl = (!filter || filter->IsTotalAllowFilter()) ? nullptr : filter; static const arrow::Schema emptySchema = arrow::Schema(arrow::FieldVector()); - TBatchIterator iterator( - batch, filterImpl, *SortSchema, (!isDenyFilter && DataSchema) ? *DataSchema : emptySchema, Reverse, VersionColumnNames, sourceId); + TBatchIterator iterator(batch, filterImpl, *SortSchema, (!isDenyFilter && DataSchema) ? *DataSchema : emptySchema, VersionColumnNames, + sourceId, TIterationOrder(Reverse, order.GetStart())); if (MaxVersion) { MaxVersion->ValidateSchema(*iterator.GetVersionColumns().GetSorting()); } diff --git a/ydb/core/formats/arrow/ut/ut_arrow.cpp b/ydb/core/formats/arrow/ut/ut_arrow.cpp index ccbf117a5eef..f531ca4f4c61 100644 --- a/ydb/core/formats/arrow/ut/ut_arrow.cpp +++ b/ydb/core/formats/arrow/ut/ut_arrow.cpp @@ -694,7 +694,7 @@ Y_UNIT_TEST_SUITE(ArrowTest) { auto merger = std::make_shared(batch->schema(), batch->schema(), false, vColumns, std::nullopt); for (auto&& i : batches) { - merger->AddSource(i, nullptr); + merger->AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Forward(0)); } merger->DrainAll(builder); sorted = builder.Finalize(); @@ -721,7 +721,7 @@ Y_UNIT_TEST_SUITE(ArrowTest) { const std::vector vColumns = {batch->schema()->field(0)->name()}; auto merger = std::make_shared(batch->schema(), batch->schema(), true, vColumns, std::nullopt); for (auto&& i : batches) { - merger->AddSource(i, nullptr); + merger->AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Reversed(0)); } merger->DrainAll(builder); sorted = builder.Finalize(); @@ -748,7 +748,7 @@ Y_UNIT_TEST_SUITE(ArrowTest) { auto merger = std::make_shared(batch->schema(), batches[0]->schema(), false, vColumns, std::nullopt); for (auto&& i : batches) { - merger->AddSource(i, nullptr); + merger->AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Forward(0)); } merger->DrainAll(builder); sorted = builder.Finalize(); @@ -787,7 +787,7 @@ Y_UNIT_TEST_SUITE(ArrowTest) { auto merger = std::make_shared(batch->schema(), batches[0]->schema(), false, vColumns, maxVersionCursor); for (auto&& i : batches) { - merger->AddSource(i, nullptr); + merger->AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Forward(0)); } merger->DrainAll(builder); sorted = builder.Finalize(); @@ -833,7 +833,7 @@ Y_UNIT_TEST_SUITE(ArrowTest) { auto merger = std::make_shared(sortingSchema, batches[0]->schema(), false, vColumns, maxVersionCursor); for (auto&& i : batches) { - merger->AddSource(i, nullptr); + merger->AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Forward(0)); } merger->DrainAll(builder); sorted = builder.Finalize(); diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 4d711600224c..0b69676d7e18 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -71,6 +71,7 @@ struct TEvPrivate { EvMetadataAccessorsInfo, EvRequestFilter, + EvFilterConstructionResult, EvEnd }; diff --git a/ydb/core/tx/columnshard/common/snapshot.h b/ydb/core/tx/columnshard/common/snapshot.h index 4f9e3841f791..7e1b7a78b148 100644 --- a/ydb/core/tx/columnshard/common/snapshot.h +++ b/ydb/core/tx/columnshard/common/snapshot.h @@ -102,6 +102,10 @@ class TSnapshot { TString DebugString() const; NJson::TJsonValue DebugJson() const; + + operator size_t() const { + return CombineHashes(PlanStep, TxId); + } }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/counters/counters_manager.h b/ydb/core/tx/columnshard/counters/counters_manager.h index 2f6b2c4ad86a..2ade7ca6770a 100644 --- a/ydb/core/tx/columnshard/counters/counters_manager.h +++ b/ydb/core/tx/columnshard/counters/counters_manager.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ class TCountersManager { YDB_READONLY(TIndexationCounters, IndexationCounters, TIndexationCounters("Indexation")); YDB_READONLY(TIndexationCounters, CompactionCounters, TIndexationCounters("GeneralCompaction")); YDB_READONLY(TScanCounters, ScanCounters, TScanCounters("Scan")); + YDB_READONLY_DEF(TDuplicateFilteringCounters, DuplicateFilteringCounters); YDB_READONLY_DEF(std::shared_ptr, RequestsTracingCounters); YDB_READONLY_DEF(std::shared_ptr, SubscribeCounters); diff --git a/ydb/core/tx/columnshard/counters/duplicate_filtering.cpp b/ydb/core/tx/columnshard/counters/duplicate_filtering.cpp new file mode 100644 index 000000000000..daf1817df9be --- /dev/null +++ b/ydb/core/tx/columnshard/counters/duplicate_filtering.cpp @@ -0,0 +1,10 @@ +#include "duplicate_filtering.h" + +namespace NKikimr::NColumnShard { +TDuplicateFilteringCounters::TDuplicateFilteringCounters() + : TBase("DuplicateFiltering") + , MergeRowsAccepted(TBase::GetDeriviative("SourcesMerging/RowsAccepted")) + , MergeRowsRejected(TBase::GetDeriviative("SourcesMerging/RowsRejected")) + , MergeRowsBulkAccepted(TBase::GetDeriviative("SourcesMerging/RowsBulkAccepted")) { +} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/duplicate_filtering.h b/ydb/core/tx/columnshard/counters/duplicate_filtering.h new file mode 100644 index 000000000000..06cda0b95c2a --- /dev/null +++ b/ydb/core/tx/columnshard/counters/duplicate_filtering.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +#include + +namespace NKikimr::NColumnShard { +class TDuplicateFilteringCounters: public TCommonCountersOwner { +private: + using TBase = TCommonCountersOwner; + + NMonitoring::TDynamicCounters::TCounterPtr MergeRowsAccepted; + NMonitoring::TDynamicCounters::TCounterPtr MergeRowsRejected; + NMonitoring::TDynamicCounters::TCounterPtr MergeRowsBulkAccepted; + +public: + TDuplicateFilteringCounters(); + + void OnRowsMerged(const ui64 accepted, const ui64 rejected, const ui64 bulkAccepted) const { + MergeRowsAccepted->Add(accepted); + MergeRowsRejected->Add(rejected); + MergeRowsBulkAccepted->Add(bulkAccepted); + } +}; +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/ya.make b/ydb/core/tx/columnshard/counters/ya.make index fd81a293fbf6..5f54a69ba72b 100644 --- a/ydb/core/tx/columnshard/counters/ya.make +++ b/ydb/core/tx/columnshard/counters/ya.make @@ -16,6 +16,7 @@ SRCS( portions.cpp writes_monitor.cpp portion_index.cpp + duplicate_filtering.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp index c22642d615b8..46e8fdccb4a1 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp @@ -255,7 +255,7 @@ std::vector TMerger::Execute(const std::shared IColumnMerger::PortionRecordIndexFieldName); batch->AddField(IColumnMerger::PortionRecordIndexField, column->BuildArray(batch->num_rows())).Validate(); } - mergeStream.AddSource(batch, Filters[idx]); + mergeStream.AddSource(batch, Filters[idx], NArrow::NMerger::TIterationOrder::Forward(0)); ++idx; } auto batchResults = mergeStream.DrainAllParts(checkPoints, indexFields); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp index a27739ffcdf0..411301cb67ac 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp @@ -133,7 +133,8 @@ TConclusionStatus TStartMergeTask::DoExecuteImpl() { if (!i->GetStageResult().IsEmpty()) { isEmpty = false; } - Merger->AddSource(rb, i->GetStageResult().GetNotAppliedFilter()); + Merger->AddSource(rb, i->GetStageResult().GetNotAppliedFilter(), + NArrow::NMerger::TIterationOrder(Context->GetCommonContext()->IsReverse(), 0)); } } AFL_VERIFY(Merger->GetSourcesCount() <= Sources.size()); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.cpp new file mode 100644 index 000000000000..b69eab2aa0c1 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.cpp @@ -0,0 +1,3 @@ +#include "common.h" + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering {} diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.h new file mode 100644 index 000000000000..13cd548bb507 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/common.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +class TColumnsData { +private: + YDB_READONLY_DEF(std::shared_ptr, Data); + YDB_READONLY_DEF(std::shared_ptr, MemoryGuard); + +public: + TColumnsData(const std::shared_ptr& data, const std::shared_ptr& memory) + : Data(data) + , MemoryGuard(memory) { + AFL_VERIFY(MemoryGuard); + } + + ui64 GetRawSize() const { + return MemoryGuard->GetMemory(); + } +}; + +class TDuplicateMapInfo { +private: + TSnapshot MaxVersion; + YDB_READONLY_DEF(ui64, Offset); + YDB_READONLY_DEF(ui64, RowsCount); + YDB_READONLY_DEF(ui64, SourceId); + +public: + TDuplicateMapInfo(const TSnapshot& maxVersion, const ui64 offset, const ui64 rowsCount, const ui64 sourceId) + : MaxVersion(maxVersion) + , Offset(offset) + , RowsCount(rowsCount) + , SourceId(sourceId) { + } + + operator size_t() const { + ui64 h = 0; + h = CombineHashes(h, (size_t)MaxVersion); + h = CombineHashes(h, Offset); + h = CombineHashes(h, RowsCount); + h = CombineHashes(h, SourceId); + return h; + } + bool operator==(const TDuplicateMapInfo& other) const { + return std::tie(MaxVersion, Offset, RowsCount, SourceId) == std::tie(other.MaxVersion, other.Offset, other.RowsCount, other.SourceId); + } + + TString DebugString() const { + return TStringBuilder() << "MaxVersion=" << MaxVersion.DebugString() << ";Offset=" << Offset << ";RowsCount=" << RowsCount + << ";SourceId=" << SourceId; + } +}; + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h index 6a386fdde4f0..51c35837a309 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/events.h @@ -1,10 +1,13 @@ #pragma once +#include "common.h" + #include #include #include #include +#include namespace NKikimr::NOlap::NReader::NSimple { class IDataSource; @@ -32,4 +35,31 @@ class TEvRequestFilter: public NActors::TEventLocal& subscriber); }; +class TEvFilterConstructionResult + : public NActors::TEventLocal { +private: + using TFilters = THashMap; + TConclusion Result; + +public: + TEvFilterConstructionResult(TConclusion&& result) + : Result(std::move(result)) { + if (Result.IsSuccess()) { + for (const auto& [info, filter] : *Result) { + AFL_VERIFY(!!filter.GetRecordsCount() && filter.GetRecordsCountVerified() == info.GetRowsCount())( + "filter", filter.GetRecordsCount().value_or(0))( + "info", info.DebugString()); + } + } + } + + const TConclusion& GetConclusion() const { + return Result; + } + + TFilters&& ExtractResult() { + return Result.DetachResult(); + } +}; + } // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp new file mode 100644 index 000000000000..59ef376ebb4f --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp @@ -0,0 +1,76 @@ +#include "merge.h" + +#include + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +class TFiltersBuilder { +private: + THashMap Filters; + YDB_READONLY(ui64, RowsAdded, 0); + YDB_READONLY(ui64, RowsSkipped, 0); + + void AddImpl(const ui64 sourceId, const bool value) { + auto* findFilter = Filters.FindPtr(sourceId); + AFL_VERIFY(findFilter); + findFilter->Add(value); + } + +public: + void AddRecord(const NArrow::NMerger::TBatchIterator& cursor) { + AddImpl(cursor.GetSourceId(), true); + ++RowsAdded; + } + + void SkipRecord(const NArrow::NMerger::TBatchIterator& cursor) { + AddImpl(cursor.GetSourceId(), false); + ++RowsSkipped; + } + + void ValidateDataSchema(const std::shared_ptr& /*schema*/) const { + } + + bool IsBufferExhausted() const { + return false; + } + + THashMap&& ExtractFilters() && { + return std::move(Filters); + } + + void AddSource(const ui64 sourceId) { + AFL_VERIFY(Filters.emplace(sourceId, NArrow::TColumnFilter::BuildAllowFilter()).second); + } +}; + +void TBuildDuplicateFilters::DoExecute(const std::shared_ptr& /*taskPtr*/) { + NArrow::NMerger::TMergePartialStream merger(PKSchema, nullptr, false, VersionColumnNames, MaxVersion); + merger.PutControlPoint(Finish.BuildSortablePosition(), false); + TFiltersBuilder filtersBuilder; + for (const auto& [interval, data] : SourcesById) { + merger.AddSource(data->GetData(), nullptr, NArrow::NMerger::TIterationOrder::Forward(interval.GetOffset()), interval.GetSourceId()); + filtersBuilder.AddSource(interval.GetSourceId()); + } + merger.DrainToControlPoint(filtersBuilder, IncludeFinish); + Counters.OnRowsMerged(filtersBuilder.GetRowsAdded(), filtersBuilder.GetRowsSkipped(), 0); + + THashMap filtersBySource = std::move(filtersBuilder).ExtractFilters(); + THashMap filters; + for (auto&& [interval, data] : SourcesById) { + NArrow::TColumnFilter* findFilter = filtersBySource.FindPtr(interval.GetSourceId()); + AFL_VERIFY(findFilter); + filters.emplace(interval, std::move(*findFilter)); + } + + AFL_VERIFY(Owner); + TActivationContext::AsActorContext().Send(Owner, new TEvFilterConstructionResult(std::move(filters))); + Owner = TActorId(); +} + +void TBuildDuplicateFilters::DoOnCannotExecute(const TString& reason) { + AFL_VERIFY(Owner); + TActivationContext::AsActorContext().Send(Owner, new TEvFilterConstructionResult(TConclusionStatus::Fail(reason))); + Owner = TActorId(); +} + +} // namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.h new file mode 100644 index 000000000000..e12dbe668631 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.h @@ -0,0 +1,56 @@ +#pragma once + +#include "common.h" + +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NReader::NSimple::NDuplicateFiltering { + +class TBuildDuplicateFilters: public NConveyor::ITask { +private: + THashMap> SourcesById; + std::shared_ptr PKSchema; + std::vector VersionColumnNames; + TActorId Owner; + NColumnShard::TDuplicateFilteringCounters Counters; + std::optional MaxVersion; + NArrow::TSimpleRow Finish; + bool IncludeFinish; + +private: + virtual void DoExecute(const std::shared_ptr& /*taskPtr*/) override; + virtual void DoOnCannotExecute(const TString& reason) override; + + virtual TString GetTaskClassIdentifier() const override { + return "BUILD_DUPLICATE_FILTERS"; + } + +public: + TBuildDuplicateFilters(const std::shared_ptr& sortingSchema, const std::optional& maxVersion, + const NArrow::TSimpleRow& finish, const bool includeFinish, const NColumnShard::TDuplicateFilteringCounters& counters, + const TActorId& owner) + : PKSchema(sortingSchema) + , VersionColumnNames(IIndexInfo::GetSnapshotColumnNames()) + , Owner(owner) + , Counters(counters) + , MaxVersion(maxVersion) + , Finish(finish) + , IncludeFinish(includeFinish) { + AFL_VERIFY(finish.GetSchema()->Equals(sortingSchema)); + } + + void AddSource(const std::shared_ptr& batch, const TDuplicateMapInfo& interval) { + AFL_VERIFY(interval.GetRowsCount()); + AFL_VERIFY(interval.GetOffset() < batch->GetData()->GetRecordsCount())("interval", interval.DebugString())( + "records", batch->GetData()->GetRecordsCount()); + AFL_VERIFY(SourcesById.emplace(interval, batch).second); + } +}; + +} // namespace NKikimr::NOlap::NReader::NSimple diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make index 3e9720055408..3f3851979bf5 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/ya.make @@ -3,6 +3,8 @@ LIBRARY() SRCS( manager.cpp events.cpp + merge.cpp + common.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp index f697f6d3a4c9..1661a6bf831c 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp @@ -160,7 +160,7 @@ class TSliceToMerge { NArrow::NMerger::TMergePartialStream stream(context.GetActualSchema()->GetIndexInfo().GetReplaceKey(), dataSchema, false, { IIndexInfo::GetWriteIdField()->name() }, std::nullopt); for (auto&& i : containers) { - stream.AddSource(i, nullptr); + stream.AddSource(i, nullptr, NArrow::NMerger::TIterationOrder::Forward(0)); } NArrow::NMerger::TRecordBatchBuilder rbBuilder(dataSchema->fields(), recordsCountSum); stream.DrainAll(rbBuilder); From bf94acd32a7579059503800f75e2d2b2d99eadce Mon Sep 17 00:00:00 2001 From: xyliganSereja Date: Fri, 6 Jun 2025 17:15:16 +0300 Subject: [PATCH 03/62] revert version normalizer to template... (#19332) --- .../portion/clean_index_columns.cpp | 1 + .../normalizer/portion/clean_index_columns.h | 42 ++++++ .../portion/clean_ttl_preset_setting_info.cpp | 1 + .../portion/clean_ttl_preset_setting_info.h | 37 ++++++ .../clean_ttl_preset_setting_version_info.cpp | 1 + .../clean_ttl_preset_setting_version_info.h | 37 ++++++ .../portion/clean_unused_tables.cpp | 1 - .../normalizer/portion/clean_unused_tables.h | 34 ----- .../portion/clean_unused_tables_template.h | 125 ++++++++---------- .../tx/columnshard/normalizer/portion/ya.make | 6 +- .../tx/columnshard/ut_rw/ut_normalizer.cpp | 18 +-- 11 files changed, 181 insertions(+), 122 deletions(-) create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.cpp create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.h create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.cpp create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.h create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.cpp create mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.h delete mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.cpp delete mode 100644 ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.h diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.cpp new file mode 100644 index 000000000000..edc0e5631910 --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.cpp @@ -0,0 +1 @@ +#include "clean_index_columns.h" diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.h b/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.h new file mode 100644 index 000000000000..51e6a8bdabeb --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_index_columns.h @@ -0,0 +1,42 @@ +#pragma once + +#include "clean_unused_tables_template.h" + +#include +#include +#include + +namespace NKikimr::NOlap::NCleanUnusedTables { +using namespace NColumnShard; + +using TUnusedIndexColumns = TCleanUnusedTablesNormalizerTemplate; + +class TCleanIndexColumnsNormalizer final: public TUnusedIndexColumns { + using TBase = TUnusedIndexColumns; + + static TString ClassName() { + return "CleanIndexColumns"; + } + + static inline auto Registrator = INormalizerComponent::TFactory::TRegistrator(ClassName()); + +public: + explicit TCleanIndexColumnsNormalizer(const TNormalizationController::TInitContext& ctx) + : TBase(ctx) { + } + + TString GetClassName() const override { + return ClassName(); + } + + std::optional DoGetEnumSequentialId() const override { + return std::nullopt; + } + +protected: + virtual bool ValidateConfig() override { + return !AppData()->ColumnShardConfig.GetColumnChunksV0Usage(); + } +}; + +} // namespace NKikimr::NOlap::NCleanUnusedTables diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.cpp new file mode 100644 index 000000000000..46461bbe7b7d --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.cpp @@ -0,0 +1 @@ +#include "clean_ttl_preset_setting_info.h" \ No newline at end of file diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.h b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.h new file mode 100644 index 000000000000..905fdc0800d6 --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_info.h @@ -0,0 +1,37 @@ +#pragma once + +#include "clean_unused_tables_template.h" + +#include +#include +#include + +namespace NKikimr::NOlap::NCleanUnusedTables { +using namespace NColumnShard; + +using TUnusedTtlPresetInfo = TCleanUnusedTablesNormalizerTemplate; + +class TCleanTtlPresetInfoNormalizer final: public TUnusedTtlPresetInfo { + using TBase = TUnusedTtlPresetInfo; + + static TString ClassName() { + return "CleanTtlPresetSettingsInfo"; + } + + static inline auto Registrator = INormalizerComponent::TFactory::TRegistrator(ClassName()); + +public: + explicit TCleanTtlPresetInfoNormalizer(const TNormalizationController::TInitContext& ctx) + : TBase(ctx) { + } + + TString GetClassName() const override { + return ClassName(); + } + + std::optional DoGetEnumSequentialId() const override { + return std::nullopt; + } +}; + +} // namespace NKikimr::NOlap::NCleanUnusedTables diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.cpp new file mode 100644 index 000000000000..277c69310f73 --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.cpp @@ -0,0 +1 @@ +#include "clean_ttl_preset_setting_version_info.h" \ No newline at end of file diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.h b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.h new file mode 100644 index 000000000000..d136716002cd --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_ttl_preset_setting_version_info.h @@ -0,0 +1,37 @@ +#pragma once + +#include "clean_unused_tables_template.h" + +#include +#include +#include + +namespace NKikimr::NOlap::NCleanUnusedTables { +using namespace NColumnShard; + +using TUnusedTtlPresetVersionInfo = TCleanUnusedTablesNormalizerTemplate; + +class TCleanTtlPresetVersionInfoNormalizer final: public TUnusedTtlPresetVersionInfo { + using TBase = TUnusedTtlPresetVersionInfo; + + static TString ClassName() { + return "CleanTtlPresetSettingsVersionInfo"; + } + + static inline auto Registrator = INormalizerComponent::TFactory::TRegistrator(ClassName()); + +public: + explicit TCleanTtlPresetVersionInfoNormalizer(const TNormalizationController::TInitContext& ctx) + : TBase(ctx) { + } + + TString GetClassName() const override { + return ClassName(); + } + + std::optional DoGetEnumSequentialId() const override { + return std::nullopt; + } +}; + +} // namespace NKikimr::NOlap::NCleanUnusedTables diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.cpp deleted file mode 100644 index 3e89b2695790..000000000000 --- a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "clean_unused_tables.h" diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.h b/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.h deleted file mode 100644 index bcfd597b2a8f..000000000000 --- a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables.h +++ /dev/null @@ -1,34 +0,0 @@ -#pragma once - -#include "clean_unused_tables_template.h" -#include -#include - -namespace NKikimr::NOlap::NCleanUnusedTables { -using namespace NColumnShard; - -using TUnusedTables = TCleanUnusedTablesNormalizerTemplate< - Schema::IndexColumns, - Schema::TtlSettingsPresetInfo, - Schema::TtlSettingsPresetVersionInfo ->; - -class TCleanUnusedTablesNormalizer final : public TUnusedTables { - using TBase = TUnusedTables; - - static TString ClassName() { return "CleanUnusedTables"; } - - static inline auto Registrator = - INormalizerComponent::TFactory::TRegistrator(ClassName()); - -public: - explicit TCleanUnusedTablesNormalizer(const TNormalizationController::TInitContext& ctx) - : TBase(ctx) {} - - TString GetClassName() const override { return ClassName(); } - std::optional DoGetEnumSequentialId() const override { - return std::nullopt; - } -}; - -} // namespace NKikimr::NOlap::NCleanUnusedTables diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables_template.h b/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables_template.h index 9dbf56b0a2bb..5b9073d737aa 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables_template.h +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_unused_tables_template.h @@ -1,123 +1,110 @@ #pragma once -#include -#include - #include #include +#include +#include +#include +#include + namespace NKikimr::NOlap::NCleanUnusedTables { using namespace NColumnShard; using NIceDb::TNiceDb; template inline void Delete(TNiceDb& db, const TKey& key) { - std::apply([&](auto... parts) { db.template Table().Key(parts...).Delete(); }, key); + std::apply( + [&](auto... parts) { + db.template Table().Key(parts...).Delete(); + }, + key); } -template -class TCleanUnusedTablesNormalizerTemplate - : public TNormalizationController::INormalizerComponent { - protected: +template +class TCleanUnusedTablesNormalizerTemplate: public TNormalizationController::INormalizerComponent { using TBase = TNormalizationController::INormalizerComponent; + static TString ClassName() { + return "CleanUnusedTables"; + } - static TString ClassName() { return "CleanUnusedTables"; } +protected: + virtual bool ValidateConfig() { + return true; + } public: static constexpr size_t BATCH = 1000; explicit TCleanUnusedTablesNormalizerTemplate(const TNormalizationController::TInitContext& ctx) - : TBase(ctx) {} - - TString GetClassName() const override { return ClassName(); } - - std::optional DoGetEnumSequentialId() const override { return std::nullopt; } - - TConclusion> DoInit(const TNormalizationController&, - NTabletFlatExecutor::TTransactionContext& txc) override { - TNiceDb db(txc.DB); - std::vector tasks; - - auto append = [&](auto&& res) -> decltype(auto) { - if (!res) { - return false; - } - - const auto& vec = res.GetResult(); - tasks.insert(tasks.end(), - std::make_move_iterator(vec.begin()), - std::make_move_iterator(vec.end())); - return true; - }; - - if (!(append(ProcessTable(db))&& ...)) { - return TConclusionStatus::Fail("Table not ready"); - } - - return tasks; + : TBase(ctx) { } -private: - template - static bool TableExists(TNiceDb& db) { - return db.HaveTable(); + TString GetClassName() const override { + return ClassName(); } - - template - INormalizerTask::TPtr MakeTask(TKeyVec&& vec) { - return std::make_shared( - std::make_shared>(std::forward(vec))); + std::optional DoGetEnumSequentialId() const override { + return std::nullopt; } - template - TConclusion> ProcessTable(TNiceDb& db) { + TConclusion> DoInit( + const TNormalizationController&, NTabletFlatExecutor::TTransactionContext& txc) override { using TKey = typename TTable::TKey::TupleType; + AFL_VERIFY(ValidateConfig()); + + TNiceDb db(txc.DB); std::vector tasks; - if (!TableExists(db)) { + if (!db.HaveTable()) { return tasks; } - std::vector keys; - keys.reserve(BATCH); + std::vector batch; + batch.reserve(BATCH); + auto rs = db.Table().Select(); if (!rs.IsReady()) { return TConclusionStatus::Fail("Table not ready"); } while (!rs.EndOfSet()) { - keys.emplace_back(rs.GetKey()); + batch.emplace_back(rs.GetKey()); - if (keys.size() == BATCH) { - tasks.emplace_back(MakeTask(std::move(keys))); - - keys.clear(); - keys.reserve(BATCH); + if (batch.size() == BATCH) { + tasks.emplace_back(MakeTask(std::move(batch))); + batch.clear(); + batch.reserve(BATCH); } if (!rs.Next()) { - return TConclusionStatus::Fail("IndexColumns iterate"); + return TConclusionStatus::Fail("Iterate error"); } } - if (!keys.empty()) { - tasks.emplace_back(MakeTask(std::move(keys))); + if (!batch.empty()) { + tasks.emplace_back(MakeTask(std::move(batch))); } return tasks; } - template - class TChanges final : public INormalizerChanges { +private: + template + static INormalizerTask::TPtr MakeTask(TVec&& v) { + return std::make_shared(std::make_shared(std::forward(v))); + } + + class TChanges final: public INormalizerChanges { using TKey = typename TTable::TKey::TupleType; std::vector Keys; - public: - explicit TChanges(std::vector&& k) : Keys(std::move(k)) {} + public: + explicit TChanges(std::vector&& v) + : Keys(std::move(v)) { + } - bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, - const TNormalizationController&) const override { + bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController&) const override { TNiceDb db(txc.DB); for (const auto& k : Keys) { Delete(db, k); @@ -126,8 +113,10 @@ class TCleanUnusedTablesNormalizerTemplate return true; } - ui64 GetSize() const override { return Keys.size(); } + ui64 GetSize() const override { + return Keys.size(); + } }; }; -} // namespace NKikimr::NOlap::NCleanUnusedTables +} // namespace NKikimr::NOlap::NCleanUnusedTables diff --git a/ydb/core/tx/columnshard/normalizer/portion/ya.make b/ydb/core/tx/columnshard/normalizer/portion/ya.make index 2d94321423b4..79196335804c 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/ya.make +++ b/ydb/core/tx/columnshard/normalizer/portion/ya.make @@ -14,8 +14,10 @@ SRCS( GLOBAL leaked_blobs.cpp GLOBAL clean_deprecated_snapshot.cpp GLOBAL chunks_v0_meta.cpp - GLOBAL clean_unused_tables.cpp - clean_unused_tables_template.cpp + GLOBAL clean_index_columns.cpp + GLOBAL clean_unused_tables_template.cpp + GLOBAL clean_ttl_preset_setting_info.cpp + GLOBAL clean_ttl_preset_setting_version_info.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index f4ee435257a2..09a74a986b2a 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -223,22 +223,6 @@ class TTrashUnusedInjector : public NYDBTest::ILocalDBModifier { .Update(); } } - - if (db.HaveTable()) { - for (size_t i = 0; i < 100; ++i) { - db.Table() - .Key(1 + i) - .Update(); - } - } - - if (db.HaveTable()) { - for (size_t i = 0; i < 100; ++i) { - db.Table() - .Key(1 + i, 2 + i, 3 + i) - .Update(); - } - } } }; @@ -374,7 +358,7 @@ Y_UNIT_TEST_SUITE(Normalizers) { public: virtual void CorrectConfigurationOnStart(NKikimrConfig::TColumnShardConfig& columnShardConfig) const override { auto* repair = columnShardConfig.MutableRepairs()->Add(); - repair->SetClassName("CleanUnusedTables"); + repair->SetClassName("CleanIndexColumns"); repair->SetDescription("Cleaning old table"); } }; From 67a71fa6580c32119a694896920b97dfb258b3a2 Mon Sep 17 00:00:00 2001 From: Vladilen Date: Fri, 6 Jun 2025 19:47:59 +0200 Subject: [PATCH 04/62] Fix Missing Step in EvReadSetAck (#19424) --- ydb/core/tx/columnshard/columnshard_impl.cpp | 2 +- .../tx/columnshard/transactions/operators/ev_write/primary.h | 2 +- .../columnshard/transactions/operators/ev_write/secondary.h | 2 +- ydb/core/tx/columnshard/transactions/tx_controller.h | 4 ++++ ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp | 3 --- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index fd5b30e6a33e..de762d37fc2c 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -1190,7 +1190,7 @@ void TColumnShard::Handle(TEvTxProcessing::TEvReadSet::TPtr& ev, const TActorCon AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "read_set_ignored")("proto", ev->Get()->Record.DebugString()); Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward( - new TEvTxProcessing::TEvReadSetAck(0, txId, TabletID(), ev->Get()->Record.GetTabletProducer(), TabletID(), 0), + new TEvTxProcessing::TEvReadSetAck(ev->Get()->Record.GetStep(), txId, TabletID(), ev->Get()->Record.GetTabletProducer(), TabletID(), 0), ev->Get()->Record.GetTabletProducer(), true), IEventHandle::FlagTrackDelivery, txId); return; diff --git a/ydb/core/tx/columnshard/transactions/operators/ev_write/primary.h b/ydb/core/tx/columnshard/transactions/operators/ev_write/primary.h index db8a15e97fef..1b83b08570d5 100644 --- a/ydb/core/tx/columnshard/transactions/operators/ev_write/primary.h +++ b/ydb/core/tx/columnshard/transactions/operators/ev_write/primary.h @@ -193,7 +193,7 @@ class TEvWriteCommitPrimaryTransactionOperator: public TEvWriteCommitSyncTransac } owner.Send(MakePipePerNodeCacheID(EPipePerNodeCache::Persistent), new TEvPipeCache::TEvForward( - new TEvTxProcessing::TEvReadSetAck(0, GetTxId(), owner.TabletID(), i, owner.TabletID(), 0), i, true), + new TEvTxProcessing::TEvReadSetAck(GetStep(), GetTxId(), owner.TabletID(), i, owner.TabletID(), 0), i, true), IEventHandle::FlagTrackDelivery, GetTxId()); } } diff --git a/ydb/core/tx/columnshard/transactions/operators/ev_write/secondary.h b/ydb/core/tx/columnshard/transactions/operators/ev_write/secondary.h index 9790eadb0fbe..751017ea501e 100644 --- a/ydb/core/tx/columnshard/transactions/operators/ev_write/secondary.h +++ b/ydb/core/tx/columnshard/transactions/operators/ev_write/secondary.h @@ -150,7 +150,7 @@ class TEvWriteCommitSecondaryTransactionOperator: public TEvWriteCommitSyncTrans void SendBrokenFlagAck(TColumnShard& owner) { owner.Send(MakePipePerNodeCacheID(EPipePerNodeCache::Persistent), new TEvPipeCache::TEvForward( - new TEvTxProcessing::TEvReadSetAck(0, GetTxId(), owner.TabletID(), ArbiterTabletId, owner.TabletID(), 0), ArbiterTabletId, true), + new TEvTxProcessing::TEvReadSetAck(GetStep(), GetTxId(), owner.TabletID(), ArbiterTabletId, owner.TabletID(), 0), ArbiterTabletId, true), IEventHandle::FlagTrackDelivery, GetTxId()); } diff --git a/ydb/core/tx/columnshard/transactions/tx_controller.h b/ydb/core/tx/columnshard/transactions/tx_controller.h index af7a497b47a9..5b9c02f9e83f 100644 --- a/ydb/core/tx/columnshard/transactions/tx_controller.h +++ b/ydb/core/tx/columnshard/transactions/tx_controller.h @@ -309,6 +309,10 @@ class TTxController { return TxInfo.TxId; } + ui64 GetStep() const { + return TxInfo.PlanStep; + } + OpType GetOpType() const { return DoGetOpType(); } diff --git a/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp b/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp index ba7f2c670ce8..237469864e0c 100644 --- a/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp +++ b/ydb/public/sdk/cpp/src/client/topic/ut/topic_to_table_ut.cpp @@ -3098,7 +3098,6 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_6_Query, TFixtureSinksQuery) void TFixtureSinks::TestSinksOlapWriteToTopicAndTable1() { - return; // https://github.com/ydb-platform/ydb/issues/17271 CreateTopic("topic_A"); CreateColumnTable("/Root/table_A"); @@ -3121,7 +3120,6 @@ void TFixtureSinks::TestSinksOlapWriteToTopicAndTable1() Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2, TFixtureSinks) { - return; // https://github.com/ydb-platform/ydb/issues/17271 CreateTopic("topic_A"); CreateTopic("topic_B"); @@ -3188,7 +3186,6 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3, TFixtureSinks) void TFixtureSinks::TestSinksOlapWriteToTopicAndTable4() { - return; // https://github.com/ydb-platform/ydb/issues/17271 CreateTopic("topic_A"); CreateTopic("topic_B"); From e9b005107e84ff65b7b971015bb0e2ef940be563 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sun, 25 May 2025 10:42:32 +0300 Subject: [PATCH 05/62] one conveyor for insert and compaction. draft. (#18776) --- ydb/core/base/events.h | 1 + ydb/core/protos/config.proto | 25 +- .../engines/portions/data_accessor.cpp | 1 + .../engines/writer/buffer/actor2.cpp | 2 +- ydb/core/tx/conveyor/usage/abstract.h | 8 + ydb/core/tx/conveyor/usage/service.h | 24 +- ydb/core/tx/conveyor/ya.make | 11 + .../conveyor_composite/service/category.cpp | 81 ++++ .../tx/conveyor_composite/service/category.h | 50 +++ .../tx/conveyor_composite/service/common.cpp | 37 ++ .../tx/conveyor_composite/service/common.h | 127 ++++++ .../conveyor_composite/service/counters.cpp | 31 ++ .../tx/conveyor_composite/service/counters.h | 79 ++++ .../tx/conveyor_composite/service/events.cpp | 23 + .../tx/conveyor_composite/service/events.h | 117 +++++ .../tx/conveyor_composite/service/manager.cpp | 4 + .../tx/conveyor_composite/service/manager.h | 78 ++++ .../tx/conveyor_composite/service/process.cpp | 5 + .../tx/conveyor_composite/service/process.h | 122 +++++ .../tx/conveyor_composite/service/scope.cpp | 38 ++ .../tx/conveyor_composite/service/scope.h | 71 +++ .../tx/conveyor_composite/service/service.cpp | 69 +++ .../tx/conveyor_composite/service/service.h | 67 +++ .../tx/conveyor_composite/service/worker.cpp | 64 +++ .../tx/conveyor_composite/service/worker.h | 69 +++ .../service/workers_pool.cpp | 85 ++++ .../conveyor_composite/service/workers_pool.h | 100 +++++ .../tx/conveyor_composite/service/ya.make | 21 + .../tx/conveyor_composite/usage/common.cpp | 18 + ydb/core/tx/conveyor_composite/usage/common.h | 44 ++ .../tx/conveyor_composite/usage/config.cpp | 206 +++++++++ ydb/core/tx/conveyor_composite/usage/config.h | 138 ++++++ .../tx/conveyor_composite/usage/events.cpp | 15 + ydb/core/tx/conveyor_composite/usage/events.h | 69 +++ .../tx/conveyor_composite/usage/service.cpp | 5 + .../tx/conveyor_composite/usage/service.h | 83 ++++ ydb/core/tx/conveyor_composite/usage/ya.make | 17 + .../tx/conveyor_composite/ut/ut_simple.cpp | 425 ++++++++++++++++++ ydb/core/tx/conveyor_composite/ut/ya.make | 30 ++ ydb/core/tx/conveyor_composite/ya.make | 15 + ydb/core/tx/ya.make | 2 + 41 files changed, 2468 insertions(+), 9 deletions(-) create mode 100644 ydb/core/tx/conveyor/ya.make create mode 100644 ydb/core/tx/conveyor_composite/service/category.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/category.h create mode 100644 ydb/core/tx/conveyor_composite/service/common.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/common.h create mode 100644 ydb/core/tx/conveyor_composite/service/counters.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/counters.h create mode 100644 ydb/core/tx/conveyor_composite/service/events.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/events.h create mode 100644 ydb/core/tx/conveyor_composite/service/manager.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/manager.h create mode 100644 ydb/core/tx/conveyor_composite/service/process.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/process.h create mode 100644 ydb/core/tx/conveyor_composite/service/scope.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/scope.h create mode 100644 ydb/core/tx/conveyor_composite/service/service.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/service.h create mode 100644 ydb/core/tx/conveyor_composite/service/worker.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/worker.h create mode 100644 ydb/core/tx/conveyor_composite/service/workers_pool.cpp create mode 100644 ydb/core/tx/conveyor_composite/service/workers_pool.h create mode 100644 ydb/core/tx/conveyor_composite/service/ya.make create mode 100644 ydb/core/tx/conveyor_composite/usage/common.cpp create mode 100644 ydb/core/tx/conveyor_composite/usage/common.h create mode 100644 ydb/core/tx/conveyor_composite/usage/config.cpp create mode 100644 ydb/core/tx/conveyor_composite/usage/config.h create mode 100644 ydb/core/tx/conveyor_composite/usage/events.cpp create mode 100644 ydb/core/tx/conveyor_composite/usage/events.h create mode 100644 ydb/core/tx/conveyor_composite/usage/service.cpp create mode 100644 ydb/core/tx/conveyor_composite/usage/service.h create mode 100644 ydb/core/tx/conveyor_composite/usage/ya.make create mode 100644 ydb/core/tx/conveyor_composite/ut/ut_simple.cpp create mode 100644 ydb/core/tx/conveyor_composite/ut/ya.make create mode 100644 ydb/core/tx/conveyor_composite/ya.make diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 4e652a4f3226..eb9010f205d3 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -185,6 +185,7 @@ struct TKikimrEvents : TEvents { ES_FEATURE_FLAGS = 4262, ES_PRIORITY_QUEUE = 4263, ES_SOLOMON_PROVIDER = 4264, + ES_CONVEYOR_COMPOSITE = 4265, }; }; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 2f34fa35c199..523dc17698fa 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -637,6 +637,28 @@ message TConveyorConfig { optional double WorkersCountDouble = 5; } +message TCompositeConveyorConfig { + message TCategory { + optional string Name = 1; + optional uint64 QueueSizeLimit = 2; + } + + message TWorkerPoolCategoryLink { + optional string Category = 1; + optional double Weight = 2; + } + + message TWorkersPool { + optional double WorkersCount = 1; + optional double DefaultFractionOfThreadsCount = 2; + repeated TWorkerPoolCategoryLink Links = 3; + } + + optional bool Enabled = 1 [default = true]; + repeated TWorkersPool WorkerPools = 2; + repeated TCategory Categories = 3; +} + message TPrioritiesQueueConfig { optional bool Enabled = 1 [default = true]; optional uint32 Limit = 2 [default = 32]; @@ -1099,7 +1121,7 @@ message TQueryServiceConfig { optional uint32 QueryTimeoutDefaultSeconds = 19 [default = 1800]; optional bool EnableMatchRecognize = 20 [default = false]; repeated string AvailableExternalDataSources = 22; // Ignored if AllExternalDataSourcesAreAvailable is true - optional bool AllExternalDataSourcesAreAvailable = 23 [default = true]; + optional bool AllExternalDataSourcesAreAvailable = 23 [default = true]; } // Config describes immediate controls and allows @@ -2333,6 +2355,7 @@ message TAppConfig { optional NKikimrBlobStorage.TYamlConfig StoredConfigYaml = 105; optional string StartupConfigYaml = 107; optional string StartupStorageYaml = 108; + optional TCompositeConveyorConfig CompositeConveyorConfig = 109; } message TYdbVersion { diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp index 07d4e6fe7090..c58ae15bed8b 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp @@ -651,6 +651,7 @@ void TPortionDataAccessor::FullValidation() const { TBlobRange::Validate(PortionInfo->GetMeta().GetBlobIds(), i.GetBlobRange()).Validate(); blobIdxs.emplace(i.GetBlobRange().GetBlobIdxVerified()); } + AFL_VERIFY(GetRecordsVerified().size()); for (auto&& i : GetIndexesVerified()) { if (auto bRange = i.GetBlobRangeOptional()) { TBlobRange::Validate(PortionInfo->GetMeta().GetBlobIds(), *bRange).Validate(); diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp index dfa57b5bee89..329efccbd77a 100644 --- a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp +++ b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp @@ -77,4 +77,4 @@ void TWriteAggregation::Flush(const ui64 tabletId) { } } -} // namespace NKikimr::NColumnShard::NWritingPortions +} // namespace NKikimr::NOlap::NWritingPortions diff --git a/ydb/core/tx/conveyor/usage/abstract.h b/ydb/core/tx/conveyor/usage/abstract.h index 909136d42639..c9fa9da33bc1 100644 --- a/ydb/core/tx/conveyor/usage/abstract.h +++ b/ydb/core/tx/conveyor/usage/abstract.h @@ -19,6 +19,14 @@ class TTaskSignals: public NColumnShard::TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr Success; NMonitoring::TDynamicCounters::TCounterPtr SuccessDuration; + TTaskSignals(const NColumnShard::TCommonCountersOwner& baseObject, const TString& taskClassIdentifier) + : TBase(baseObject, "task_class_name", taskClassIdentifier) { + Fails = TBase::GetDeriviative("Fails"); + FailsDuration = TBase::GetDeriviative("FailsDuration"); + Success = TBase::GetDeriviative("Success"); + SuccessDuration = TBase::GetDeriviative("SuccessDuration"); + } + TTaskSignals(const TString& moduleId, const TString& taskClassIdentifier, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals = nullptr) : TBase(moduleId, baseSignals) { DeepSubGroup("task_class", taskClassIdentifier); diff --git a/ydb/core/tx/conveyor/usage/service.h b/ydb/core/tx/conveyor/usage/service.h index 2acd2ede377d..a943ec29b62a 100644 --- a/ydb/core/tx/conveyor/usage/service.h +++ b/ydb/core/tx/conveyor/usage/service.h @@ -1,20 +1,21 @@ #pragma once #include "config.h" -#include -#include + #include #include +#include +#include + namespace NKikimr::NConveyor { class TAsyncTaskExecutor: public TActorBootstrapped { private: const std::shared_ptr Task; + public: TAsyncTaskExecutor(const std::shared_ptr& task) - : Task(task) - { - + : Task(task) { } void Bootstrap() { @@ -23,6 +24,12 @@ class TAsyncTaskExecutor: public TActorBootstrapped { } }; +enum class ESpecialTaskProcesses { + Insert = 1, + Compaction = 2, + Normalizer = 3 +}; + template class TServiceOperatorImpl { private: @@ -35,11 +42,15 @@ class TServiceOperatorImpl { Y_ABORT_UNLESS(TConveyorPolicy::Name.size() == 4); return TConveyorPolicy::Name; } + public: static void AsyncTaskToExecute(const std::shared_ptr& task) { auto& context = NActors::TActorContext::AsActorContext(); context.Register(new TAsyncTaskExecutor(task)); } + static bool SendTaskToExecute(const std::shared_ptr& task, const ESpecialTaskProcesses processType) { + return SendTaskToExecute(task, (ui64)processType); + } static bool SendTaskToExecute(const std::shared_ptr& task, const ui64 processId = 0) { if (TSelf::IsEnabled() && NActors::TlsActivationContext) { auto& context = NActors::TActorContext::AsActorContext(); @@ -71,7 +82,6 @@ class TServiceOperatorImpl { return TProcessGuard(externalProcessId, {}); } } - }; class TScanConveyorPolicy { @@ -96,4 +106,4 @@ using TScanServiceOperator = TServiceOperatorImpl; using TCompServiceOperator = TServiceOperatorImpl; using TInsertServiceOperator = TServiceOperatorImpl; -} +} // namespace NKikimr::NConveyor diff --git a/ydb/core/tx/conveyor/ya.make b/ydb/core/tx/conveyor/ya.make new file mode 100644 index 000000000000..e5c415537d4d --- /dev/null +++ b/ydb/core/tx/conveyor/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( +) + +PEERDIR( + ydb/core/tx/conveyor/service + ydb/core/tx/conveyor/usage +) + +END() diff --git a/ydb/core/tx/conveyor_composite/service/category.cpp b/ydb/core/tx/conveyor_composite/service/category.cpp new file mode 100644 index 000000000000..4c87ae85d0d8 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/category.cpp @@ -0,0 +1,81 @@ +#include "category.h" + +namespace NKikimr::NConveyorComposite { + +bool TProcessCategory::HasTasks() const { + for (auto&& i : Scopes) { + if (i.second->HasTasks()) { + return true; + } + } + return false; +} + +void TProcessCategory::DoQuant(const TMonotonic newStart) { + CPUUsage->Cut(newStart); + for (auto&& i : Scopes) { + i.second->DoQuant(newStart); + } +} + +TWorkerTask TProcessCategory::ExtractTaskWithPrediction() { + std::shared_ptr scopeMin; + TDuration dMin; + for (auto&& [_, scope] : Scopes) { + if (!scope->HasTasks()) { + continue; + } + const TDuration d = scope->GetCPUUsage()->CalcWeight(scope->GetWeight()); + if (!scopeMin || d < dMin) { + dMin = d; + scopeMin = scope; + } + } + AFL_VERIFY(scopeMin); + return scopeMin->ExtractTaskWithPrediction(); +} + +TProcessScope& TProcessCategory::MutableProcessScope(const TString& scopeName) { + auto it = Scopes.find(scopeName); + AFL_VERIFY(it != Scopes.end())("cat", GetCategory())("scope", scopeName); + return *it->second; +} + +TProcessScope* TProcessCategory::MutableProcessScopeOptional(const TString& scopeName) { + auto it = Scopes.find(scopeName); + if (it != Scopes.end()) { + return it->second.get(); + } else { + return nullptr; + } +} + +TProcessScope& TProcessCategory::RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { + TCPUGroup::TPtr cpuGroup = std::make_shared(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); + auto info = Scopes.emplace(scopeId, std::make_shared(std::move(cpuGroup), CPUUsage)); + AFL_VERIFY(info.second); + return *info.first->second; +} + +TProcessScope& TProcessCategory::UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { + auto& scope = MutableProcessScope(scopeId); + scope.UpdateLimits(processCpuLimits); + return scope; +} + +TProcessScope& TProcessCategory::UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { + if (Scopes.contains(scopeId)) { + return UpdateScope(scopeId, processCpuLimits); + } else { + return RegisterScope(scopeId, processCpuLimits); + } +} + +void TProcessCategory::UnregisterScope(const TString& name) { + auto it = Scopes.find(name); + AFL_VERIFY(it != Scopes.end()); + AFL_VERIFY(!it->second->GetProcessesCount()); + Scopes.erase(it); +} + +} diff --git a/ydb/core/tx/conveyor_composite/service/category.h b/ydb/core/tx/conveyor_composite/service/category.h new file mode 100644 index 000000000000..78c8f9b6b61e --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/category.h @@ -0,0 +1,50 @@ +#pragma once +#include "common.h" +#include "counters.h" +#include "scope.h" +#include "worker.h" + +#include + +namespace NKikimr::NConveyorComposite { + +class TProcessCategory: public TNonCopyable { +private: + const ESpecialTaskCategory Category; + std::shared_ptr CPUUsage = std::make_shared(nullptr); + TPositiveControlInteger WaitingTasksCount; + YDB_READONLY_DEF(std::shared_ptr, Counters); + THashMap> Scopes; + const NConfig::TCategory Config; + +public: + TProcessCategory(const NConfig::TCategory& config, TCounters& counters) + : Category(config.GetCategory()) + , Config(config) { + Counters = counters.GetCategorySignals(Category); + } + + ESpecialTaskCategory GetCategory() const { + return Category; + } + + void PutTaskResult(TWorkerTaskResult&& result) { + const TString id = result.GetScopeId(); + if (TProcessScope* scope = MutableProcessScopeOptional(id)) { + scope->PutTaskResult(std::move(result)); + } + } + + bool HasTasks() const; + void DoQuant(const TMonotonic newStart); + TWorkerTask ExtractTaskWithPrediction(); + TProcessScope& MutableProcessScope(const TString& scopeName); + TProcessScope* MutableProcessScopeOptional(const TString& scopeName); + TProcessScope& RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + TProcessScope& UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + + TProcessScope& UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + void UnregisterScope(const TString& name); +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/common.cpp b/ydb/core/tx/conveyor_composite/service/common.cpp new file mode 100644 index 000000000000..46a9129dc635 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/common.cpp @@ -0,0 +1,37 @@ +#include "common.h" + +#include + +namespace NKikimr::NConveyorComposite { + +void TCPUUsage::Exchange(const TDuration predicted, const TMonotonic start, const TMonotonic finish) { + Usage.emplace_back(TTaskCPUUsage(start, finish)); + AFL_VERIFY(predicted <= PredictedDuration)("predicted_delta", predicted)("predicted_sum", PredictedDuration); + Duration += Usage.back().GetDuration(); + PredictedDuration -= predicted; + if (Parent) { + Parent->Exchange(predicted, start, finish); + } +} + +void TCPUUsage::Cut(const TMonotonic start) { + ui32 idx = 0; + while (idx < Usage.size()) { + AFL_VERIFY(Usage[idx].GetDuration() <= Duration); + Duration -= Usage[idx].GetDuration(); + if (Usage[idx].GetFinish() <= start) { + std::swap(Usage[idx], Usage.back()); + Usage.pop_back(); + } else { + Usage[idx].Cut(start); + Duration += Usage[idx].GetDuration(); + ++idx; + } + } +} + +TCPUGroup::~TCPUGroup() { + AFL_VERIFY(ProcessesCount == 0); +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/common.h b/ydb/core/tx/conveyor_composite/service/common.h new file mode 100644 index 000000000000..b35e49a17f49 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/common.h @@ -0,0 +1,127 @@ +#pragma once +#include +#include +#include +#include + +#include + +namespace NKikimr::NConveyorComposite { + +class TTaskCPUUsage { +private: + YDB_READONLY_DEF(TMonotonic, Start); + YDB_READONLY_DEF(TMonotonic, Finish); + YDB_READONLY_DEF(TDuration, Duration); + +public: + void Cut(const TMonotonic start) { + AFL_VERIFY(start < Finish); + if (Start <= start) { + Start = start; + } + } + + TTaskCPUUsage(const TMonotonic start, const TMonotonic finish) + : Start(start) + , Finish(finish) + , Duration(finish - start) { + } +}; + +template +class TAverageCalcer { +private: + const ui32 Count = 100; + std::deque Values; + T Sum = T(); + +public: + TAverageCalcer(const ui32 count = 100) + : Count(count) { + } + + void Add(const T value) { + Values.emplace_back(value); + Sum += value; + if (Values.size() > Count) { + Sum -= Values.front(); + Values.pop_front(); + } + } + + T GetValue() const { + return Values.size() ? (Sum / Values.size()) : T(); + } +}; + +class TCPUUsage { +private: + std::deque Usage; + YDB_READONLY_DEF(TDuration, Duration); + YDB_READONLY_DEF(TDuration, PredictedDuration); + std::shared_ptr Parent; + +public: + TCPUUsage(const std::shared_ptr& parent) + : Parent(parent) { + } + + TDuration CalcWeight(const double w) const { + if (w <= 0) { + return TDuration::Max(); + } else { + return (Duration + PredictedDuration) * w; + } + } + + void AddPredicted(const TDuration d) { + PredictedDuration += d; + if (Parent) { + Parent->AddPredicted(d); + } + } + + void AddUsage(const TTaskCPUUsage& usage) { + Usage.emplace_back(usage); + Duration += usage.GetDuration(); + if (Parent) { + Parent->AddUsage(usage); + } + } + + void Exchange(const TDuration predicted, const TMonotonic start, const TMonotonic finish); + void Cut(const TMonotonic start); +}; + +class TCPUGroup { + YDB_ACCESSOR_DEF(double, CPUThreadsLimit); + YDB_ACCESSOR(double, Weight, 1); + TPositiveControlInteger ProcessesCount; + +public: + using TPtr = std::shared_ptr; + + TCPUGroup(const double cpuThreadsLimit, const double weight = 1) + : CPUThreadsLimit(cpuThreadsLimit) + , Weight(weight) + { + } + + ~TCPUGroup(); + + ui32 GetProcessesCount() const { + return ProcessesCount.Val(); + } + + bool DecProcesses() { + --ProcessesCount; + return ProcessesCount == 0; + } + + void IncProcesses() { + ++ProcessesCount; + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/counters.cpp b/ydb/core/tx/conveyor_composite/service/counters.cpp new file mode 100644 index 000000000000..59e2f7020887 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/counters.cpp @@ -0,0 +1,31 @@ +#include "counters.h" + +namespace NKikimr::NConveyorComposite { + +TCounters::TCounters(const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals) + : TBase("CompositeConveyor/" + conveyorName, baseSignals) + , ProcessesCount(TBase::GetValue("Processes/Count")) + , WaitingQueueSize(TBase::GetValue("WaitingQueueSize")) + , WaitingQueueSizeLimit(TBase::GetValue("WaitingQueueSizeLimit")) + , AvailableWorkersCount(TBase::GetValue("AvailableWorkersCount")) + , WorkersCountLimit(TBase::GetValue("WorkersCountLimit")) + , AmountCPULimit(TBase::GetValue("AmountCPULimit")) + , IncomingRate(TBase::GetDeriviative("Incoming")) + , SolutionsRate(TBase::GetDeriviative("Solved")) + , OverlimitRate(TBase::GetDeriviative("Overlimit")) + , WaitWorkerRate(TBase::GetDeriviative("WaitWorker")) + , UseWorkerRate(TBase::GetDeriviative("UseWorker")) + , ChangeCPULimitRate(TBase::GetDeriviative("ChangeCPULimit")) + , WaitingHistogram(TBase::GetHistogram("Waiting/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , PackHistogram(TBase::GetHistogram("ExecutionPack/Count", NMonitoring::LinearHistogram(25, 1, 1))) + , PackExecuteHistogram(TBase::GetHistogram("PackExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , TaskExecuteHistogram(TBase::GetHistogram("TaskExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendBackHistogram(TBase::GetHistogram("SendBack/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendFwdHistogram(TBase::GetHistogram("SendForward/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , ReceiveTaskHistogram(TBase::GetHistogram("ReceiveTask/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendBackDuration(TBase::GetDeriviative("SendBack/Duration/Us")) + , SendFwdDuration(TBase::GetDeriviative("SendForward/Duration/Us")) + , ExecuteDuration(TBase::GetDeriviative("Execute/Duration/Us")) { +} + +} diff --git a/ydb/core/tx/conveyor_composite/service/counters.h b/ydb/core/tx/conveyor_composite/service/counters.h new file mode 100644 index 000000000000..e6b7f38fc8f2 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/counters.h @@ -0,0 +1,79 @@ +#pragma once +#include +#include + +#include + +namespace NKikimr::NConveyorComposite { + +using TTaskSignals = NConveyor::TTaskSignals; + +class TCategorySignals: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + THashMap> TaskClassSignals; + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + +public: + TCategorySignals(NColumnShard::TCommonCountersOwner& base, const ESpecialTaskCategory cat) + : TBase(base, "category", ::ToString(cat)) + , Category(cat) { + } + + std::shared_ptr GetTaskSignals(const TString& taskClassName) { + auto it = TaskClassSignals.find(taskClassName); + if (it == TaskClassSignals.end()) { + it = TaskClassSignals.emplace(taskClassName, std::make_shared(*this, taskClassName)).first; + } + return it->second; + } +}; + +class TCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + THashMap> CategorySignals; + +public: + const ::NMonitoring::TDynamicCounters::TCounterPtr ProcessesCount; + + const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSize; + const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSizeLimit; + + const ::NMonitoring::TDynamicCounters::TCounterPtr InProgressSize; + + const ::NMonitoring::TDynamicCounters::TCounterPtr AvailableWorkersCount; + const ::NMonitoring::TDynamicCounters::TCounterPtr WorkersCountLimit; + const ::NMonitoring::TDynamicCounters::TCounterPtr AmountCPULimit; + + const ::NMonitoring::TDynamicCounters::TCounterPtr IncomingRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr SolutionsRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr OverlimitRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr WaitWorkerRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr UseWorkerRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr ChangeCPULimitRate; + + const ::NMonitoring::THistogramPtr WaitingHistogram; + const ::NMonitoring::THistogramPtr PackHistogram; + const ::NMonitoring::THistogramPtr PackExecuteHistogram; + const ::NMonitoring::THistogramPtr TaskExecuteHistogram; + const ::NMonitoring::THistogramPtr SendBackHistogram; + const ::NMonitoring::THistogramPtr SendFwdHistogram; + const ::NMonitoring::THistogramPtr ReceiveTaskHistogram; + + const ::NMonitoring::TDynamicCounters::TCounterPtr SendBackDuration; + const ::NMonitoring::TDynamicCounters::TCounterPtr SendFwdDuration; + const ::NMonitoring::TDynamicCounters::TCounterPtr ExecuteDuration; + + std::shared_ptr GetCategorySignals(const ESpecialTaskCategory cat) { + auto it = CategorySignals.find(cat); + if (it == CategorySignals.end()) { + it = CategorySignals.emplace(cat, std::make_shared(*this, cat)).first; + } + return it->second; + } + + TCounters(const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals); +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/events.cpp b/ydb/core/tx/conveyor_composite/service/events.cpp new file mode 100644 index 000000000000..68c63c027ca1 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/events.cpp @@ -0,0 +1,23 @@ +#include "events.h" + +#include + +namespace NKikimr::NConveyorComposite { + +TEvInternal::TEvTaskProcessedResult::TEvTaskProcessedResult( + std::vector&& results, const TDuration forwardSendDuration, const ui64 workerIdx, const ui64 workersPoolId) + : ForwardSendDuration(forwardSendDuration) + , Results(std::move(results)) + , WorkerIdx(workerIdx) + , WorkersPoolId(workersPoolId) { + AFL_VERIFY(Results.size()); +} + +TWorkerTaskResult::TWorkerTaskResult(const TWorkerTaskContext& context, const TMonotonic start, const TMonotonic finish) + : TBase(context) + , Start(start) + , Finish(finish) { + AFL_VERIFY(Start <= Finish); +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/events.h b/ydb/core/tx/conveyor_composite/service/events.h new file mode 100644 index 000000000000..dcafb80c6755 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/events.h @@ -0,0 +1,117 @@ +#pragma once +#include "counters.h" + +#include + +#include +#include +#include +#include + +namespace NKikimr::NConveyorComposite { + +class TWorkerTaskContext { +private: + YDB_READONLY(TMonotonic, CreateInstant, TMonotonic::Now()); + YDB_READONLY_DEF(TDuration, PredictedDuration); + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY_DEF(TString, ScopeId); + YDB_READONLY(ui64, ProcessId, 0); + +public: + TWorkerTaskContext(const TDuration prediction, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + : PredictedDuration(prediction) + , Category(category) + , ScopeId(scopeId) + , ProcessId(processId) { + } +}; + +class TWorkerTask; + +class TWorkerTaskResult: public TWorkerTaskContext { +private: + using TBase = TWorkerTaskContext; + YDB_READONLY_DEF(TMonotonic, Start); + YDB_READONLY_DEF(TMonotonic, Finish); + + TWorkerTaskResult(const TWorkerTaskContext& context, const TMonotonic start, const TMonotonic finish); + friend class TWorkerTask; + +public: + TDuration GetDuration() const { + return Finish - Start; + } + +}; + +class TWorkerTask: public TWorkerTaskContext { +private: + using TBase = TWorkerTaskContext; + YDB_READONLY_DEF(ITask::TPtr, Task); + YDB_READONLY_DEF(std::shared_ptr, TaskSignals); + +public: + TWorkerTaskResult GetResult(const TMonotonic start, const TMonotonic finish) const { + return TWorkerTaskResult(*this, start, finish); + } + + TWorkerTask(const ITask::TPtr& task, const TDuration prediction, const ESpecialTaskCategory category, const TString& scopeId, + const std::shared_ptr& taskSignals, const ui64 processId) + : TBase(prediction, category, scopeId, processId) + , Task(task) + , TaskSignals(taskSignals) { + Y_ABORT_UNLESS(task); + } + + bool operator<(const TWorkerTask& wTask) const { + return Task->GetPriority() < wTask.Task->GetPriority(); + } +}; + +struct TEvInternal { + enum EEv { + EvNewTask = EventSpaceBegin(NActors::TEvents::ES_PRIVATE), + EvTaskProcessedResult, + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(NActors::TEvents::ES_PRIVATE), "expected EvEnd < EventSpaceEnd"); + + class TEvNewTask: public NActors::TEventLocal { + private: + std::vector Tasks; + YDB_READONLY(TMonotonic, ConstructInstant, TMonotonic::Now()); + + public: + TEvNewTask() = default; + + std::vector&& ExtractTasks() { + return std::move(Tasks); + } + + explicit TEvNewTask(std::vector&& tasks) + : Tasks(std::move(tasks)) { + } + }; + + class TEvTaskProcessedResult: public NActors::TEventLocal { + private: + using TBase = TConclusion; + YDB_READONLY_DEF(TDuration, ForwardSendDuration); + std::vector Results; + YDB_READONLY(TMonotonic, ConstructInstant, TMonotonic::Now()); + YDB_READONLY(ui64, WorkerIdx, 0); + YDB_READONLY(ui64, WorkersPoolId, 0); + + public: + std::vector&& DetachResults() { + return std::move(Results); + } + + TEvTaskProcessedResult( + std::vector&& results, const TDuration forwardSendDuration, const ui64 workerIdx, const ui64 workersPoolId); + }; +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/manager.cpp b/ydb/core/tx/conveyor_composite/service/manager.cpp new file mode 100644 index 000000000000..d62b06ae24a3 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/manager.cpp @@ -0,0 +1,4 @@ +#include "category.h" + +namespace NKikimr::NConveyorComposite { +} diff --git a/ydb/core/tx/conveyor_composite/service/manager.h b/ydb/core/tx/conveyor_composite/service/manager.h new file mode 100644 index 000000000000..30c0cc7666f6 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/manager.h @@ -0,0 +1,78 @@ +#pragma once +#include "category.h" +#include "workers_pool.h" + +#include + +namespace NKikimr::NConveyorComposite { +class TTasksManager { +private: + const TString ConveyorName; + std::vector> WorkerPools; + std::vector> Categories; + const NActors::TActorId DistributorActorId; + const NConfig::TConfig Config; + TCounters& Counters; + +public: + TString DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << ConveyorName << ":"; + for (auto&& wp : WorkerPools) { + sb << wp->GetMaxWorkerThreads() << ","; + } + sb << ";"; + sb << "}"; + return sb; + } + + void DoQuant(const TMonotonic newStart) { + for (auto&& c : Categories) { + c->DoQuant(newStart); + } + } + + TTasksManager(const TString& convName, const NConfig::TConfig& config, const NActors::TActorId distributorActorId, TCounters& counters) + : ConveyorName(convName) + , DistributorActorId(distributorActorId) + , Config(config) + , Counters(counters) + { + for (auto&& i : GetEnumAllValues()) { + Categories.emplace_back(std::make_shared(Config.GetCategoryConfig(i), Counters)); + } + for (auto&& i : Config.GetWorkerPools()) { + WorkerPools.emplace_back(std::make_shared(ConveyorName, distributorActorId, i, Counters, Categories)); + } + } + + TWorkersPool& MutableWorkersPool(const ui32 workersPoolId) { + AFL_VERIFY(workersPoolId < WorkerPools.size()); + return *WorkerPools[workersPoolId]; + } + + [[nodiscard]] bool DrainTasks() { + bool result = false; + for (auto&& i : WorkerPools) { + if (i->DrainTasks()) { + result = true; + } + } + return result; + } + + TProcessCategory& MutableCategoryVerified(const ESpecialTaskCategory category) { + AFL_VERIFY((ui64)category < Categories.size()); + AFL_VERIFY(!!Categories[(ui64)category]); + return *Categories[(ui64)category]; + } + + const TProcessCategory& GetCategoryVerified(const ESpecialTaskCategory category) const { + AFL_VERIFY((ui64)category < Categories.size()); + AFL_VERIFY(!!Categories[(ui64)category]); + return *Categories[(ui64)category]; + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/process.cpp b/ydb/core/tx/conveyor_composite/service/process.cpp new file mode 100644 index 000000000000..b4bf9a8611f3 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/process.cpp @@ -0,0 +1,5 @@ +#include "process.h" + +namespace NKikimr::NConveyorComposite { + +} diff --git a/ydb/core/tx/conveyor_composite/service/process.h b/ydb/core/tx/conveyor_composite/service/process.h new file mode 100644 index 000000000000..cbc58bb4de47 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/process.h @@ -0,0 +1,122 @@ +#pragma once +#include "common.h" +#include "worker.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include + +namespace NKikimr::NConveyorComposite { + +class TDequePriorityFIFO { +private: + std::map> Tasks; + ui32 Size = 0; + +public: + void push(const TWorkerTask& task) { + Tasks[(ui32)task.GetTask()->GetPriority()].emplace_back(task); + ++Size; + } + TWorkerTask pop() { + Y_ABORT_UNLESS(Size); + auto result = std::move(Tasks.rbegin()->second.front()); + Tasks.rbegin()->second.pop_front(); + if (Tasks.rbegin()->second.size() == 0) { + Tasks.erase(--Tasks.end()); + } + --Size; + return result; + } + ui32 size() const { + return Size; + } +}; + +class TProcessOrdered { +private: + YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, CPUTime, 0); + +public: + TProcessOrdered(const ui64 processId, const ui64 cpuTime) + : ProcessId(processId) + , CPUTime(cpuTime) { + } + + bool operator<(const TProcessOrdered& item) const { + if (CPUTime < item.CPUTime) { + return true; + } + if (item.CPUTime < CPUTime) { + return false; + } + return ProcessId < item.ProcessId; + } +}; + +class TProcess: public TMoveOnly { +private: + YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY_DEF(std::shared_ptr, CPUUsage); + YDB_ACCESSOR_DEF(TDequePriorityFIFO, Tasks); + TAverageCalcer AverageTaskDuration; + ui32 LinksCount = 0; + +public: + void DoQuant(const TMonotonic newStart) { + CPUUsage->Cut(newStart); + } + + bool HasTasks() const { + return Tasks.size(); + } + + TWorkerTask ExtractTaskWithPrediction() { + auto result = Tasks.pop(); + CPUUsage->AddPredicted(result.GetPredictedDuration()); + return result; + } + + void PutTaskResult(TWorkerTaskResult&& result) { + CPUUsage->Exchange(result.GetPredictedDuration(), result.GetStart(), result.GetFinish()); + AverageTaskDuration.Add(result.GetDuration()); + } + + [[nodiscard]] bool DecRegistration() { + AFL_VERIFY(LinksCount); + --LinksCount; + return LinksCount == 0; + } + + double GetWeight() const { + return 1.0; + } + + void IncRegistration() { + ++LinksCount; + } + + TProcess(const ui64 processId, const std::shared_ptr& scopeUsage) + : ProcessId(processId) + { + CPUUsage = std::make_shared(scopeUsage); + IncRegistration(); + } + + void RegisterTask(const std::shared_ptr& task, const TString& scopeId, const std::shared_ptr& signals) { + TWorkerTask wTask(task, AverageTaskDuration.GetValue(), signals->GetCategory(), scopeId, + signals->GetTaskSignals(task->GetTaskClassIdentifier()), ProcessId); + Tasks.push(std::move(wTask)); + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/scope.cpp b/ydb/core/tx/conveyor_composite/service/scope.cpp new file mode 100644 index 000000000000..0613ee69e20d --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/scope.cpp @@ -0,0 +1,38 @@ +#include "scope.h" + +namespace NKikimr::NConveyorComposite { + +bool TProcessScope::HasTasks() const { + for (auto&& i : Processes) { + if (i.second.HasTasks()) { + return true; + } + } + return false; +} + +TWorkerTask TProcessScope::ExtractTaskWithPrediction() { + TProcess* pMin = nullptr; + TDuration dMin = TDuration::Max(); + for (auto&& [_, p] : Processes) { + if (!p.HasTasks()) { + continue; + } + const TDuration d = p.GetCPUUsage()->CalcWeight(p.GetWeight()); + if (!pMin || d < dMin) { + pMin = &p; + dMin = d; + } + } + AFL_VERIFY(pMin)("size", Processes.size()); + return pMin->ExtractTaskWithPrediction(); +} + +void TProcessScope::DoQuant(const TMonotonic newStart) { + CPUUsage->Cut(newStart); + for (auto&& i : Processes) { + i.second.DoQuant(newStart); + } +} + +} diff --git a/ydb/core/tx/conveyor_composite/service/scope.h b/ydb/core/tx/conveyor_composite/service/scope.h new file mode 100644 index 000000000000..b95a15dad202 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/scope.h @@ -0,0 +1,71 @@ +#pragma once +#include "common.h" +#include "process.h" + +namespace NKikimr::NConveyorComposite { + +class TProcessScope: public TNonCopyable { +private: + YDB_READONLY_DEF(std::shared_ptr, CPUUsage); + TCPUGroup::TPtr ScopeLimits; + THashMap Processes; + +public: + double GetWeight() const { + return ScopeLimits->GetWeight(); + } + + ui32 GetProcessesCount() const { + return Processes.size(); + } + bool HasTasks() const; + + TWorkerTask ExtractTaskWithPrediction(); + + void DoQuant(const TMonotonic newStart); + + void UpdateLimits(const TCPULimitsConfig& processCpuLimits) { + ScopeLimits->SetCPUThreadsLimit(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); + ScopeLimits->SetWeight(processCpuLimits.GetWeight()); + } + + void PutTaskResult(TWorkerTaskResult&& result) { + const ui64 id = result.GetProcessId(); + if (auto* process = MutableProcessOptional(id)) { + process->PutTaskResult(std::move(result)); + } + } + + TProcessScope(TCPUGroup::TPtr&& limits, const std::shared_ptr& categoryScope) + : CPUUsage(std::make_shared(categoryScope)) + , ScopeLimits(std::move(limits)) { + } + + TProcess& MutableProcessVerified(const ui64 processId) { + auto it = Processes.find(processId); + AFL_VERIFY(it != Processes.end()); + return it->second; + } + + TProcess* MutableProcessOptional(const ui64 processId) { + auto it = Processes.find(processId); + if (it != Processes.end()) { + return &it->second; + } else { + return nullptr; + } + } + + void RegisterProcess(const ui64 processId) { + TProcess process(processId, CPUUsage); + AFL_VERIFY(Processes.emplace(processId, std::move(process)).second); + ScopeLimits->IncProcesses(); + } + + bool UnregisterProcess(const ui64 processId) { + AFL_VERIFY(Processes.erase(processId)); + return ScopeLimits->DecProcesses(); + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/service.cpp b/ydb/core/tx/conveyor_composite/service/service.cpp new file mode 100644 index 000000000000..1195f7c1a081 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/service.cpp @@ -0,0 +1,69 @@ +#include "manager.h" +#include "service.h" + +#include +#include + +namespace NKikimr::NConveyorComposite { + +TDistributor::TDistributor( + const NConfig::TConfig& config, const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) + : Config(config) + , ConveyorName(conveyorName) + , Counters(ConveyorName, conveyorSignals) { +} + +TDistributor::~TDistributor() { +} + +void TDistributor::Bootstrap() { + Manager = std::make_unique(ConveyorName, Config, SelfId(), Counters); + AFL_NOTICE(NKikimrServices::TX_CONVEYOR)("name", ConveyorName)("action", "conveyor_registered")("config", Config.DebugString())( + "actor_id", SelfId())("manager", Manager->DebugString()); + Become(&TDistributor::StateMain); + TBase::Schedule(TDuration::Seconds(1), new NActors::TEvents::TEvWakeup(1)); +} + +void TDistributor::HandleMain(NActors::TEvents::TEvWakeup::TPtr& evExt) { + if (evExt->Get()->Tag == 1) { + Manager->DoQuant(TMonotonic::Now()); + TBase::Schedule(TDuration::Seconds(1), new NActors::TEvents::TEvWakeup(1)); + } +} + +void TDistributor::HandleMain(TEvInternal::TEvTaskProcessedResult::TPtr& evExt) { + TWorkersPool& workersPool = Manager->MutableWorkersPool(evExt->Get()->GetWorkersPoolId()); + workersPool.AddDeliveryDuration(evExt->Get()->GetForwardSendDuration() + (TMonotonic::Now() - evExt->Get()->GetConstructInstant())); + workersPool.ReleaseWorker(evExt->Get()->GetWorkerIdx()); + for (auto&& i : evExt->Get()->DetachResults()) { + workersPool.PutTaskResult(std::move(i)); + } + if (workersPool.HasTasks()) { + AFL_VERIFY(workersPool.DrainTasks()); + } +} + +void TDistributor::HandleMain(TEvExecution::TEvRegisterProcess::TPtr& ev) { + Manager->MutableCategoryVerified(ev->Get()->GetCategory()) + .UpsertScope(ev->Get()->GetScopeId(), ev->Get()->GetCPULimits()) + .RegisterProcess(ev->Get()->GetProcessId()); +} + +void TDistributor::HandleMain(TEvExecution::TEvUnregisterProcess::TPtr& ev) { + auto* evData = ev->Get(); + if (Manager->MutableCategoryVerified(evData->GetCategory()) + .MutableProcessScope(evData->GetScopeId()) + .UnregisterProcess(evData->GetProcessId())) { + Manager->MutableCategoryVerified(evData->GetCategory()).UnregisterScope(evData->GetScopeId()); + } +} + +void TDistributor::HandleMain(TEvExecution::TEvNewTask::TPtr& ev) { + auto& cat = Manager->MutableCategoryVerified(ev->Get()->GetCategory()); + cat.MutableProcessScope(ev->Get()->GetScopeId()) + .MutableProcessVerified(ev->Get()->GetProcessId()) + .RegisterTask(ev->Get()->GetTask(), ev->Get()->GetScopeId(), cat.GetCounters()); + Y_UNUSED(Manager->DrainTasks()); +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/service.h b/ydb/core/tx/conveyor_composite/service/service.h new file mode 100644 index 000000000000..159fff0e4168 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/service.h @@ -0,0 +1,67 @@ +#pragma once +#include "counters.h" +#include "events.h" + +#include +#include + +#include +#include +#include +#include + +#include + +#include + +namespace NKikimr::NConveyorComposite { +class TTasksManager; +class TDistributor: public TActorBootstrapped { +private: + using TBase = TActorBootstrapped; + const NConfig::TConfig Config; + const TString ConveyorName = "common"; + std::shared_ptr Manager; + TCounters Counters; + TMonotonic LastAddProcessInstant = TMonotonic::Now(); + + void HandleMain(TEvExecution::TEvNewTask::TPtr& ev); + void HandleMain(NActors::TEvents::TEvWakeup::TPtr& ev); + void HandleMain(TEvExecution::TEvRegisterProcess::TPtr& ev); + void HandleMain(TEvExecution::TEvUnregisterProcess::TPtr& ev); + void HandleMain(TEvInternal::TEvTaskProcessedResult::TPtr& ev); + + void AddProcess(const ui64 processId, const TCPULimitsConfig& cpuLimits); + + void AddCPUTime(const ui64 processId, const TDuration d); + + TWorkerTask PopTask(); + + void PushTask(const TWorkerTask& task); + + void ChangeAmountCPULimit(const double delta); + +public: + STATEFN(StateMain) { + // NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("name", ConveyorName) + // ("workers", Workers.size())("waiting", Waiting.size())("actor_id", SelfId()); + switch (ev->GetTypeRewrite()) { + hFunc(TEvExecution::TEvNewTask, HandleMain); + hFunc(NActors::TEvents::TEvWakeup, HandleMain); + hFunc(TEvInternal::TEvTaskProcessedResult, HandleMain); + hFunc(TEvExecution::TEvRegisterProcess, HandleMain); + hFunc(TEvExecution::TEvUnregisterProcess, HandleMain); + default: + AFL_ERROR(NKikimrServices::TX_CONVEYOR)("problem", "unexpected event for task executor")("ev_type", ev->GetTypeName()); + break; + } + } + + TDistributor(const NConfig::TConfig& config, const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals); + + ~TDistributor(); + + void Bootstrap(); +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/worker.cpp b/ydb/core/tx/conveyor_composite/service/worker.cpp new file mode 100644 index 000000000000..3ab805fe3ee7 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/worker.cpp @@ -0,0 +1,64 @@ +#include "worker.h" + +namespace NKikimr::NConveyorComposite { + +TDuration TWorker::GetWakeupDuration() const { + AFL_VERIFY(ExecutionDuration); + return (*ExecutionDuration) * (1 - CPUSoftLimit) / CPUSoftLimit; +} + +void TWorker::ExecuteTask(std::vector&& workerTasks) { + AFL_VERIFY(!ExecutionDuration && Results.empty()); + std::vector results; + results.reserve(workerTasks.size()); + const TMonotonic startGlobal = TMonotonic::Now(); + for (auto&& t : workerTasks) { + const TMonotonic start = TMonotonic::Now(); + t.GetTask()->Execute(t.GetTaskSignals(), t.GetTask()); + results.emplace_back(t.GetResult(start, TMonotonic::Now())); + } + if (CPUSoftLimit < 1) { + AFL_DEBUG(NKikimrServices::TX_CONVEYOR)("action", "to_wait_result")("id", SelfId())("count", workerTasks.size()); + ExecutionDuration = TMonotonic::Now() - startGlobal; + Results = std::move(results); + Schedule(GetWakeupDuration(), new NActors::TEvents::TEvWakeup(CPULimitGeneration)); + WaitWakeUp = true; + } else { + AFL_VERIFY(!!ForwardDuration); + AFL_DEBUG(NKikimrServices::TX_CONVEYOR)("action", "to_result")("id", SelfId())("count", Results.size())("d", TMonotonic::Now() - startGlobal); + TBase::Sender(std::move(results), *ForwardDuration, WorkerIdx, WorkersPoolId).SendTo(DistributorId); + ForwardDuration.reset(); + } +} + +void TWorker::HandleMain(NActors::TEvents::TEvWakeup::TPtr& ev) { + const auto evGeneration = ev->Get()->Tag; + AFL_VERIFY(evGeneration <= CPULimitGeneration); + if (evGeneration == CPULimitGeneration) { + OnWakeup(); + } +} + +void TWorker::OnWakeup() { + AFL_VERIFY(ExecutionDuration); + AFL_VERIFY(Results.size()); + AFL_VERIFY(!!ForwardDuration); + AFL_DEBUG(NKikimrServices::TX_CONVEYOR)("action", "wake_up")("id", SelfId())("count", Results.size()); + TBase::Sender(std::move(Results), *ForwardDuration, WorkerIdx, WorkersPoolId).SendTo(DistributorId); + ForwardDuration.reset(); + Results.clear(); + ExecutionDuration.reset(); + + WaitWakeUp = false; +} + +void TWorker::HandleMain(TEvInternal::TEvNewTask::TPtr& ev) { + AFL_VERIFY(!WaitWakeUp); + const TMonotonic now = TMonotonic::Now(); + ForwardDuration = now - ev->Get()->GetConstructInstant(); + SendFwdHistogram->Collect(ForwardDuration->MicroSeconds()); + SendFwdDuration->Add(ForwardDuration->MicroSeconds()); + ExecuteTask(ev->Get()->ExtractTasks()); +} + +} diff --git a/ydb/core/tx/conveyor_composite/service/worker.h b/ydb/core/tx/conveyor_composite/service/worker.h new file mode 100644 index 000000000000..c8209c1e7975 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/worker.h @@ -0,0 +1,69 @@ +#pragma once + +#include "counters.h" +#include "events.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NKikimr::NConveyorComposite { + +class TWorker: public NActors::TActorBootstrapped { +private: + using TBase = NActors::TActorBootstrapped; + const double CPUHardLimit = 1; + YDB_READONLY(double, CPUSoftLimit, 1); + ui64 CPULimitGeneration = 0; + bool WaitWakeUp = false; + std::optional ForwardDuration; + const NActors::TActorId DistributorId; + const ui64 WorkerIdx; + const ui64 WorkersPoolId; + std::optional ExecutionDuration; + std::vector Results; + const ::NMonitoring::THistogramPtr SendFwdHistogram; + const ::NMonitoring::TDynamicCounters::TCounterPtr SendFwdDuration; + TDuration GetWakeupDuration() const; + void ExecuteTask(std::vector&& workerTasks); + void HandleMain(TEvInternal::TEvNewTask::TPtr& ev); + void HandleMain(NActors::TEvents::TEvWakeup::TPtr& ev); + void OnWakeup(); + +public: + STATEFN(StateMain) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvInternal::TEvNewTask, HandleMain); + hFunc(NActors::TEvents::TEvWakeup, HandleMain); + default: + ALS_ERROR(NKikimrServices::TX_CONVEYOR) << "unexpected event for task executor: " << ev->GetTypeRewrite(); + break; + } + } + + void Bootstrap() { + Become(&TWorker::StateMain); + } + + TWorker(const TString& conveyorName, const double cpuHardLimit, const NActors::TActorId& distributorId, const ui64 workerIdx, + const ui64 workersPoolId, const ::NMonitoring::THistogramPtr sendFwdHistogram, + const ::NMonitoring::TDynamicCounters::TCounterPtr sendFwdDuration) + : TBase("CONVEYOR::" + conveyorName + "::WORKER") + , CPUHardLimit(cpuHardLimit) + , CPUSoftLimit(cpuHardLimit) + , DistributorId(distributorId) + , WorkerIdx(workerIdx) + , WorkersPoolId(workersPoolId) + , SendFwdHistogram(sendFwdHistogram) + , SendFwdDuration(sendFwdDuration) { + AFL_VERIFY(0 < CPUHardLimit); + AFL_VERIFY(CPUHardLimit <= 1); + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp new file mode 100644 index 000000000000..8a7221869536 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp @@ -0,0 +1,85 @@ +#include "workers_pool.h" + +namespace NKikimr::NConveyorComposite { +TWorkersPool::TWorkersPool(const TString& conveyorName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, + const TCounters& counters, const std::vector>& categories) + : WorkersCount(config.GetWorkersCount()) + , Counters(counters) { + Workers.reserve(WorkersCount); + for (auto&& i : config.GetLinks()) { + AFL_VERIFY((ui64)i.GetCategory() < categories.size()); + Processes.emplace_back(TWeightedCategory(i.GetWeight(), categories[(ui64)i.GetCategory()])); + } + AFL_VERIFY(Processes.size()); + for (ui32 i = 0; i < WorkersCount; ++i) { + Workers.emplace_back(std::make_unique(conveyorName, config.GetWorkerCPUUsage(i), distributorId, i, + config.GetWorkersPoolId(), Counters.SendFwdHistogram, Counters.SendFwdDuration)); + ActiveWorkersIdx.emplace_back(i); + } + AFL_VERIFY(WorkersCount)("name", conveyorName)("action", "conveyor_registered")("config", config.DebugString())("actor_id", distributorId)( + "count", WorkersCount); + Counters.WaitingQueueSizeLimit->Set(config.GetWorkersCountDouble()); + Counters.AmountCPULimit->Set(0); + Counters.AvailableWorkersCount->Set(0); + Counters.WorkersCountLimit->Set(WorkersCount); +} + +bool TWorkersPool::HasFreeWorker() const { + return !ActiveWorkersIdx.empty(); +} + +void TWorkersPool::RunTask(std::vector&& tasksBatch) { + AFL_VERIFY(HasFreeWorker()); + const auto workerIdx = ActiveWorkersIdx.back(); + ActiveWorkersIdx.pop_back(); + Counters.AvailableWorkersCount->Set(ActiveWorkersIdx.size()); + + auto& worker = Workers[workerIdx]; + worker.OnStartTask(); + TActivationContext::Send(worker.GetWorkerId(), std::make_unique(std::move(tasksBatch))); +} + +void TWorkersPool::ReleaseWorker(const ui32 workerIdx) { + AFL_VERIFY(workerIdx < Workers.size()); + Workers[workerIdx].OnStopTask(); + ActiveWorkersIdx.emplace_back(workerIdx); + Counters.AvailableWorkersCount->Set(ActiveWorkersIdx.size()); +} + +bool TWorkersPool::DrainTasks() { + if (ActiveWorkersIdx.empty()) { + return false; + } + const auto predHeap = [](const TWeightedCategory& l, const TWeightedCategory& r) { + const bool hasL = l.GetCategory()->HasTasks(); + const bool hasR = r.GetCategory()->HasTasks(); + if (!hasL && !hasR) { + return false; + } else if (!hasL && hasR) { + return true; + } else if (hasL && !hasR) { + return false; + } + return r.GetCPUUsage()->CalcWeight(r.GetWeight()) < l.GetCPUUsage()->CalcWeight(l.GetWeight()); + }; + std::make_heap(Processes.begin(), Processes.end(), predHeap); + AFL_VERIFY(Processes.size()); + bool newTask = false; + while (ActiveWorkersIdx.size() && Processes.front().GetCategory()->HasTasks()) { + TDuration predicted = TDuration::Zero(); + std::vector tasks; + while ((tasks.empty() || predicted < DeliveringDuration.GetValue() * 10) && Processes.front().GetCategory()->HasTasks()) { + std::pop_heap(Processes.begin(), Processes.end(), predHeap); + tasks.emplace_back(Processes.back().GetCategory()->ExtractTaskWithPrediction()); + Processes.back().GetCPUUsage()->AddPredicted(tasks.back().GetPredictedDuration()); + predicted += tasks.back().GetPredictedDuration(); + std::push_heap(Processes.begin(), Processes.end(), predHeap); + } + newTask = true; + AFL_VERIFY(tasks.size()); + RunTask(std::move(tasks)); + } + return newTask; +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.h b/ydb/core/tx/conveyor_composite/service/workers_pool.h new file mode 100644 index 000000000000..6fa9def6c9bd --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.h @@ -0,0 +1,100 @@ +#pragma once +#include "category.h" +#include "common.h" +#include "worker.h" + +#include + +namespace NKikimr::NConveyorComposite { + +class TWeightedCategory { +private: + YDB_READONLY(std::shared_ptr, CPUUsage, std::make_shared(nullptr)); + YDB_READONLY_DEF(std::shared_ptr, Category); + YDB_READONLY(double, Weight, 1); + +public: + TWeightedCategory(const double weight, const std::shared_ptr& cat) + : Category(cat) + , Weight(weight) + { + AFL_VERIFY(cat); + AFL_VERIFY(Weight); + } +}; + +class TWorkersPool { +private: + class TWorkerInfo { + YDB_READONLY(bool, RunningTask, false); + YDB_READONLY(TWorker*, Worker, nullptr); + YDB_READONLY_DEF(NActors::TActorId, WorkerId); + + public: + explicit TWorkerInfo(std::unique_ptr&& worker) + : Worker(worker.get()) + , WorkerId(TActivationContext::Register(worker.release())) { + } + + void OnStartTask() { + AFL_VERIFY(!RunningTask); + RunningTask = true; + } + + void OnStopTask() { + AFL_VERIFY(RunningTask); + RunningTask = false; + } + }; + + YDB_READONLY(ui32, WorkersCount, 0); + YDB_READONLY(double, MaxWorkerThreads, 0); + YDB_READONLY(double, AmountCPULimit, 0); + std::vector Processes; + std::vector Workers; + std::vector ActiveWorkersIdx; + TCounters Counters; + TAverageCalcer DeliveringDuration; + std::deque DeliveryDurations; + +public: + static constexpr double Eps = 1e-6; + using TPtr = std::shared_ptr; + + TWorkersPool(const TString& conveyorName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, + const TCounters& counters, const std::vector>& categories); + + bool HasTasks() const { + for (auto&& i : Processes) { + if (i.GetCategory()->HasTasks()) { + return true; + } + } + return false; + } + + [[nodiscard]] bool DrainTasks(); + + void AddDeliveryDuration(const TDuration d) { + DeliveringDuration.Add(d); + } + + void PutTaskResult(TWorkerTaskResult&& result) { +// const ui32 catIdx = (ui32)result.GetCategory(); + for (auto&& i : Processes) { + if (i.GetCategory()->GetCategory() == result.GetCategory()) { +// AFL_VERIFY(catIdx < Processes.size()); +// AFL_VERIFY(Processes[catIdx]); + i.GetCPUUsage()->Exchange(result.GetPredictedDuration(), result.GetStart(), result.GetFinish()); + i.GetCategory()->PutTaskResult(std::move(result)); + return; + } + } + AFL_VERIFY(false); + } + bool HasFreeWorker() const; + void RunTask(std::vector&& tasksBatch); + void ReleaseWorker(const ui32 workerIdx); +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/ya.make b/ydb/core/tx/conveyor_composite/service/ya.make new file mode 100644 index 000000000000..5ca718306738 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/service/ya.make @@ -0,0 +1,21 @@ +LIBRARY() + +SRCS( + worker.cpp + service.cpp + process.cpp + common.cpp + manager.cpp + workers_pool.cpp + category.cpp + scope.cpp + counters.cpp + events.cpp +) + +PEERDIR( + ydb/core/tx/conveyor_composite/usage + ydb/core/protos +) + +END() diff --git a/ydb/core/tx/conveyor_composite/usage/common.cpp b/ydb/core/tx/conveyor_composite/usage/common.cpp new file mode 100644 index 000000000000..aeaaaa4af88e --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/common.cpp @@ -0,0 +1,18 @@ +#include "common.h" +#include "events.h" + +#include +#include + +namespace NKikimr::NConveyorComposite { + +void TProcessGuard::Finish() { + AFL_VERIFY(!Finished); + Finished = true; + if (ServiceActorId && NActors::TlsActivationContext) { + auto& context = NActors::TActorContext::AsActorContext(); + context.Send(*ServiceActorId, new TEvExecution::TEvUnregisterProcess(Category, ScopeId, ProcessId)); + } +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/common.h b/ydb/core/tx/conveyor_composite/usage/common.h new file mode 100644 index 000000000000..612b3aa511e8 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/common.h @@ -0,0 +1,44 @@ +#pragma once +#include + +namespace NKikimr::NConveyorComposite { +using ITask = NConveyor::ITask; + +enum class ESpecialTaskCategory { + Insert = 0 /* "insert" */, + Compaction = 1 /* "compaction" */, + Normalizer = 2 /* "normalizer" */, + Scan = 3 /* "scan" */ +}; + +class TProcessGuard: TNonCopyable { +private: + const ESpecialTaskCategory Category; + const TString ScopeId; + const ui64 ProcessId; + bool Finished = false; + const std::optional ServiceActorId; + +public: + ui64 GetProcessId() const { + return ProcessId; + } + + explicit TProcessGuard( + const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId, const std::optional& actorId) + : Category(category) + , ScopeId(scopeId) + , ProcessId(processId) + , ServiceActorId(actorId) { + } + + void Finish(); + + ~TProcessGuard() { + if (!Finished) { + Finish(); + } + } +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/config.cpp b/ydb/core/tx/conveyor_composite/usage/config.cpp new file mode 100644 index 000000000000..e0fbda75b34d --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/config.cpp @@ -0,0 +1,206 @@ +#include "config.h" + +#include + +#include +#include + +namespace NKikimr::NConveyorComposite::NConfig { + +TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config, const ui64 usableThreadsCount) { + if (!config.HasEnabled()) { + EnabledFlag = true; + } else { + EnabledFlag = config.GetEnabled(); + } + for (auto&& i : GetEnumAllValues()) { + Categories.emplace_back(TCategory(i)); + } + TWorkersPool* defWorkersPool = nullptr; + WorkerPools.reserve(1 + config.GetWorkerPools().size()); + if ((ui32)config.GetCategories().size() != GetEnumAllValues().size()) { + TWorkersPool wp(WorkerPools.size(), usableThreadsCount); + WorkerPools.emplace_back(std::move(wp)); + defWorkersPool = &WorkerPools.front(); + } + std::set usedCategories; + for (auto&& i : config.GetCategories()) { + TCategory cat(ESpecialTaskCategory::Insert); + auto conclusion = cat.DeserializeFromProto(i); + if (conclusion.IsFail()) { + return conclusion; + } + if (!usedCategories.emplace(cat.GetCategory()).second) { + return TConclusionStatus::Fail("category " + ::ToString(cat.GetCategory()) + " duplication"); + } + Categories[(ui64)cat.GetCategory()] = std::move(cat); + } + for (auto&& i : Categories) { + if (i.IsDefault()) { + AFL_VERIFY(defWorkersPool); + AFL_VERIFY(defWorkersPool->AddLink(i.GetCategory())); + AFL_VERIFY(i.AddWorkerPool(defWorkersPool->GetWorkersPoolId())); + } + } + for (auto&& i : config.GetWorkerPools()) { + TWorkersPool wp(WorkerPools.size()); + auto conclusion = wp.DeserializeFromProto(i, usableThreadsCount); + if (conclusion.IsFail()) { + return conclusion; + } + WorkerPools.emplace_back(std::move(wp)); + for (auto&& link : WorkerPools.back().GetLinks()) { + AFL_VERIFY((ui64)link.GetCategory() < Categories.size()); + auto& cat = Categories[(ui64)link.GetCategory()]; + if (!cat.AddWorkerPool(WorkerPools.back().GetWorkersPoolId())) { + return TConclusionStatus::Fail("double link for category: " + ::ToString(link.GetCategory())); + } + } + } + for (auto&& c : Categories) { + if (c.GetWorkerPools().empty()) { + return TConclusionStatus::Fail("no worker pools for category: " + ::ToString(c.GetCategory())); + } + } + return TConclusionStatus::Success(); +} + +double TWorkersPool::GetWorkerCPUUsage(const ui32 workerIdx) const { + AFL_VERIFY(WorkersCountDouble); + double wholePart; + const double fractionalPart = std::modf(WorkersCountDouble, &wholePart); + if (workerIdx + 1 <= wholePart) { + return 1; + } else { + AFL_VERIFY(workerIdx == wholePart); + AFL_VERIFY(fractionalPart)("count", WorkersCountDouble); + return fractionalPart; + } +} + +const TCategory& TConfig::GetCategoryConfig(const ESpecialTaskCategory cat) const { + AFL_VERIFY((ui64)cat < Categories.size()); + return Categories[(ui64)cat]; +} + +TString TConfig::DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << "{Categories:["; + for (auto&& c : Categories) { + sb << c.DebugString() << ";"; + } + sb << "]};"; + sb << "{WorkerPools:["; + for (auto&& wp : WorkerPools) { + sb << wp.DebugString() << ";"; + } + sb << "]};"; + sb << "Enabled=" << EnabledFlag << ";"; + sb << "}"; + return sb; +} + +TWorkersPool::TWorkersPool(const ui32 wpId, const double workersCountDouble) + : WorkersPoolId(wpId) + , WorkersCountDouble(workersCountDouble) { + AFL_VERIFY(WorkersCountDouble); +} + +TConclusionStatus TWorkersPool::DeserializeFromProto( + const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto, const ui64 usableThreadsCount) { + if (!proto.GetLinks().size()) { + return TConclusionStatus::Fail("no categories for workers pool"); + } + for (auto&& c : proto.GetLinks()) { + TWorkerPoolCategoryUsage link; + auto conclusion = link.DeserializeFromProto(c); + if (conclusion.IsFail()) { + return conclusion; + } + Links.emplace_back(std::move(link)); + } + if (Links.empty()) { + return TConclusionStatus::Fail("no links for workers pool"); + } + if (proto.HasWorkersCount()) { + WorkersCountDouble = proto.GetWorkersCount(); + } else if (proto.HasDefaultFractionOfThreadsCount()) { + WorkersCountDouble = usableThreadsCount * proto.GetDefaultFractionOfThreadsCount(); + } else { + WorkersCountDouble = usableThreadsCount; + } + if (WorkersCountDouble <= 0) { + return TConclusionStatus::Fail( + "incorrect WorkersCount calculated: " + proto.DebugString() + " for " + ::ToString(usableThreadsCount) + " threads"); + } + + return TConclusionStatus::Success(); +} + +TString TWorkersPool::DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << "id=" << WorkersPoolId << ";"; + sb << "count=" << WorkersCountDouble << ";"; + TStringBuilder sbLinks; + sbLinks << "["; + for (auto&& l : Links) { + sbLinks << l.DebugString() << ";"; + } + sbLinks << "]"; + sb << "links=" << sbLinks << ";"; + sb << "}"; + return sb; +} + +ui32 TWorkersPool::GetWorkersCount() const { + AFL_VERIFY(WorkersCountDouble); + return ceil(WorkersCountDouble); +} + +TString TCategory::DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << "category=" << Category << ";"; + sb << "queue_limit=" << QueueSizeLimit << ";"; + sb << "pools=" << JoinSeq(",", WorkerPools) << ";"; + sb << "}"; + return sb; +} + +TString TWorkerPoolCategoryUsage::DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << "c=" << Category << ";"; + sb << "w=" << Weight << ";"; + sb << "}"; + return sb; +} + +} // namespace NKikimr::NConveyorComposite::NConfig + +namespace NKikimr::NConveyorComposite { +TCPULimitsConfig::TCPULimitsConfig(const double cpuGroupThreadsLimit, const double weight) + : CPUGroupThreadsLimit(cpuGroupThreadsLimit) + , Weight(weight) { +} + +TConclusionStatus TCPULimitsConfig::DeserializeFromProto(const NKikimrTxDataShard::TEvKqpScan& config) { + if (config.HasCpuGroupThreadsLimit()) { + CPUGroupThreadsLimit = config.GetCpuGroupThreadsLimit(); + } + return TConclusionStatus::Success(); +} + +TString TCPULimitsConfig::DebugString() const { + TStringBuilder sb; + if (CPUGroupThreadsLimit) { + sb << "CPUGroupThreadsLimit=" << *CPUGroupThreadsLimit << ";"; + } else { + sb << "Disabled;"; + } + return sb; +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/config.h b/ydb/core/tx/conveyor_composite/usage/config.h new file mode 100644 index 000000000000..2f074b24bb2f --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/config.h @@ -0,0 +1,138 @@ +#pragma once +#include "common.h" + +#include +#include +#include + +#include +#include + +namespace NKikimr::NConveyorComposite::NConfig { + +class TWorkerPoolCategoryUsage { +private: + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY(double, Weight, 1); + +public: + TWorkerPoolCategoryUsage() = default; + + TWorkerPoolCategoryUsage(const ESpecialTaskCategory cat) + : Category(cat) { + } + + TString DebugString() const; + + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& proto) { + if (!TryFromString(proto.GetCategory(), Category)) { + return TConclusionStatus::Fail("cannot parse category link: " + proto.GetCategory()); + } + if (proto.HasWeight()) { + if (proto.GetWeight() <= 0) { + return TConclusionStatus::Fail("incorrect category link weight: " + ::ToString(proto.GetWeight())); + } + Weight = proto.GetWeight(); + } + return TConclusionStatus::Success(); + } +}; + +class TWorkersPool { +private: + YDB_READONLY(ui32, WorkersPoolId, 0); + YDB_READONLY(double, WorkersCountDouble, 0); + YDB_READONLY_DEF(std::vector, Links); + +public: + double GetWorkerCPUUsage(const ui32 workerIdx) const; + ui32 GetWorkersCount() const; + + bool AddLink(const ESpecialTaskCategory cat) { + for (auto&& i : Links) { + if (i.GetCategory() == cat) { + return false; + } + } + Links.emplace_back(TWorkerPoolCategoryUsage(cat)); + return true; + } + + TString DebugString() const; + + TWorkersPool(const ui32 wpId) + : WorkersPoolId(wpId) { + } + + TWorkersPool(const ui32 wpId, const double workersCountDouble); + + [[nodiscard]] TConclusionStatus DeserializeFromProto( + const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto, const ui64 usableThreadsCount); +}; + +class TCategory { +private: + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY(ui32, QueueSizeLimit, 256 * 1024); + YDB_READONLY_DEF(std::vector, WorkerPools); + YDB_READONLY_FLAG(Default, true); + +public: + TString DebugString() const; + + [[nodiscard]] bool AddWorkerPool(const ui32 id) { + for (auto&& i : WorkerPools) { + if (i == id) { + return false; + } + } + WorkerPools.emplace_back(id); + return true; + } + + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TCategory& proto) { + if (!TryFromString(proto.GetName(), Category)) { + return TConclusionStatus::Fail("cannot parse category: " + proto.GetName()); + } + if (proto.HasQueueSizeLimit()) { + QueueSizeLimit = proto.GetQueueSizeLimit(); + } + DefaultFlag = false; + return TConclusionStatus::Success(); + } + + TCategory(const ESpecialTaskCategory cat) + : Category(cat) { + } +}; + +class TConfig { +private: + YDB_READONLY_DEF(std::vector, Categories); + YDB_READONLY_DEF(std::vector, WorkerPools); + YDB_READONLY_FLAG(Enabled, true); + +public: + const TCategory& GetCategoryConfig(const ESpecialTaskCategory cat) const; + + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config, const ui64 usableThreadsCount); + + TString DebugString() const; +}; + +} // namespace NKikimr::NConveyorComposite::NConfig + +namespace NKikimr::NConveyorComposite { +class TCPULimitsConfig { + YDB_OPT(double, CPUGroupThreadsLimit); + YDB_READONLY(double, Weight, 1); + +public: + TCPULimitsConfig() = default; + TCPULimitsConfig(const double cpuGroupThreadsLimit, const double weight = 1); + + TConclusionStatus DeserializeFromProto(const NKikimrTxDataShard::TEvKqpScan& config); + TString DebugString() const; +}; + +} diff --git a/ydb/core/tx/conveyor_composite/usage/events.cpp b/ydb/core/tx/conveyor_composite/usage/events.cpp new file mode 100644 index 000000000000..5c94bd68be2c --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/events.cpp @@ -0,0 +1,15 @@ +#include "events.h" + +#include + +namespace NKikimr::NConveyorComposite { + +TEvExecution::TEvNewTask::TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + : Task(task) + , Category(category) + , ScopeId(scopeId) + , ProcessId(processId) { + AFL_VERIFY(Task); +} + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/events.h b/ydb/core/tx/conveyor_composite/usage/events.h new file mode 100644 index 000000000000..a3169dad9bd1 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/events.h @@ -0,0 +1,69 @@ +#pragma once +#include "common.h" +#include "config.h" + +#include + +#include +#include +#include + +namespace NKikimr::NConveyorComposite { + +struct TEvExecution { + enum EEv { + EvNewTask = EventSpaceBegin(TKikimrEvents::ES_CONVEYOR_COMPOSITE), + EvRegisterProcess, + EvUnregisterProcess, + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_CONVEYOR_COMPOSITE), "expected EvEnd < EventSpaceEnd"); + + class TEvNewTask: public NActors::TEventLocal { + private: + YDB_READONLY_DEF(ITask::TPtr, Task); + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY_DEF(TString, ScopeId); + YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(TMonotonic, ConstructInstant, TMonotonic::Now()); + + public: + TEvNewTask() = default; + + explicit TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId); + }; + + class TEvRegisterProcess: public NActors::TEventLocal { + private: + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY_DEF(TString, ScopeId); + YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY_DEF(TCPULimitsConfig, CPULimits); + + public: + explicit TEvRegisterProcess( + const TCPULimitsConfig& cpuLimits, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + : Category(category) + , ScopeId(scopeId) + , ProcessId(processId) + , CPULimits(cpuLimits) { + } + }; + + class TEvUnregisterProcess: public NActors::TEventLocal { + private: + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + YDB_READONLY_DEF(TString, ScopeId); + YDB_READONLY(ui64, ProcessId, 0); + + public: + explicit TEvUnregisterProcess(const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + : Category(category) + , ScopeId(scopeId) + , ProcessId(processId) { + } + }; +}; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/service.cpp b/ydb/core/tx/conveyor_composite/usage/service.cpp new file mode 100644 index 000000000000..eb3953901e4b --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/service.cpp @@ -0,0 +1,5 @@ +#include "service.h" + +namespace NKikimr::NConveyorComposite { + +} diff --git a/ydb/core/tx/conveyor_composite/usage/service.h b/ydb/core/tx/conveyor_composite/usage/service.h new file mode 100644 index 000000000000..336b5b5b9da0 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/service.h @@ -0,0 +1,83 @@ +#pragma once +#include "common.h" +#include "config.h" + +#include +#include + +#include +#include + +namespace NKikimr::NConveyorComposite { +template +class TServiceOperatorImpl { +private: + using TSelf = TServiceOperatorImpl; + std::atomic IsEnabledFlag = false; + static void Register(const NConfig::TConfig& serviceConfig) { + Singleton()->IsEnabledFlag = serviceConfig.IsEnabled(); + } + static const TString& GetConveyorName() { + Y_ABORT_UNLESS(TConveyorPolicy::Name.size() == 4); + return TConveyorPolicy::Name; + } + +public: + static bool SendTaskToExecute(const std::shared_ptr& task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) { + if (TSelf::IsEnabled() && NActors::TlsActivationContext) { + auto& context = NActors::TActorContext::AsActorContext(); + const NActors::TActorId& selfId = context.SelfID; + context.Send(MakeServiceId(selfId.NodeId()), new NConveyorComposite::TEvExecution::TEvNewTask(task, category, scopeId, processId)); + return true; + } else { + task->Execute(nullptr, task); + return false; + } + } + static bool IsEnabled() { + return Singleton()->IsEnabledFlag; + } + static NActors::TActorId MakeServiceId(const ui32 nodeId) { + return NActors::TActorId(nodeId, "SrvcConv" + GetConveyorName()); + } + static NActors::IActor* CreateService(const NConfig::TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) { + Register(config); + return new TDistributor(config, GetConveyorName(), conveyorSignals); + } + static TProcessGuard StartProcess( + const ESpecialTaskCategory category, const TString& scopeId, const ui64 externalProcessId, const TCPULimitsConfig& cpuLimits) { + if (TSelf::IsEnabled() && NActors::TlsActivationContext) { + auto& context = NActors::TActorContext::AsActorContext(); + const NActors::TActorId& selfId = context.SelfID; + context.Send(MakeServiceId(selfId.NodeId()), + new NConveyorComposite::TEvExecution::TEvRegisterProcess(cpuLimits, category, scopeId, externalProcessId)); + return TProcessGuard(category, scopeId, externalProcessId, MakeServiceId(selfId.NodeId())); + } else { + return TProcessGuard(category, scopeId, externalProcessId, {}); + } + } +}; + +class TScanConveyorPolicy { +public: + static const inline TString Name = "Scan"; + static constexpr bool EnableProcesses = true; +}; + +class TCompConveyorPolicy { +public: + static const inline TString Name = "Comp"; + static constexpr bool EnableProcesses = false; +}; + +class TInsertConveyorPolicy { +public: + static const inline TString Name = "Isrt"; + static constexpr bool EnableProcesses = false; +}; + +using TScanServiceOperator = TServiceOperatorImpl; +using TCompServiceOperator = TServiceOperatorImpl; +using TInsertServiceOperator = TServiceOperatorImpl; + +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/ya.make b/ydb/core/tx/conveyor_composite/usage/ya.make new file mode 100644 index 000000000000..5e3071116240 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/usage/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +SRCS( + events.cpp + config.cpp + service.cpp + common.cpp +) + +PEERDIR( + ydb/library/actors/core + ydb/services/metadata/request +) + +GENERATE_ENUM_SERIALIZATION(common.h) + +END() diff --git a/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp b/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp new file mode 100644 index 000000000000..046394f3c71c --- /dev/null +++ b/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp @@ -0,0 +1,425 @@ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +using namespace NKikimr::NConveyorComposite; + +namespace NKikimr { + +THolder BuildActorSystemSetup(const ui32 threads, const ui32 pools) { + Y_ABORT_UNLESS(threads > 0 && threads < 100); + Y_ABORT_UNLESS(pools > 0 && pools < 10); + + auto setup = MakeHolder(); + + setup->NodeId = 1; + + setup->ExecutorsCount = pools; + setup->Executors.Reset(new TAutoPtr[pools]); + for (ui32 idx : xrange(pools)) { + setup->Executors[idx] = new NActors::TBasicExecutorPool(idx, threads, 50); + } + + setup->Scheduler = new NActors::TBasicSchedulerThread(NActors::TSchedulerConfig(512, 0)); + + return setup; +} + +} // namespace NKikimr + +class TSleepTask: public NKikimr::NConveyor::ITask { +private: + const TDuration ExecutionTime; + TAtomicCounter* Counter; + virtual void DoExecute(const std::shared_ptr& /*taskPtr*/) override { + const TMonotonic start = TMonotonic::Now(); + while (TMonotonic::Now() - start < ExecutionTime) { + } + Counter->Inc(); + } + +public: + virtual TString GetTaskClassIdentifier() const override { + return "SLEEP"; + } + + TSleepTask(const TDuration d, TAtomicCounter& c) + : ExecutionTime(d) + , Counter(&c) { + } +}; + +class IRequestProcessor { +private: + YDB_READONLY_DEF(TString, Id); + virtual void DoInitialize(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) = 0; + virtual void DoAddTask(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) = 0; + virtual bool DoCheckFinished() = 0; + virtual void DoFinish(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId, const TDuration d) = 0; + virtual TString DoDebugString() const = 0; + +public: + void Initialize(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) { + DoInitialize(actorSystem, distributorId); + } + void AddTask(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) { + DoAddTask(actorSystem, distributorId); + } + bool CheckFinished() { + return DoCheckFinished(); + } + void Finish(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId, const TDuration d) { + DoFinish(actorSystem, distributorId, d); + } + TString DebugString() const { + return TStringBuilder() << "{" << Id << ":" << DoDebugString() << "}"; + } + IRequestProcessor(const TString& id) + : Id(id) { + } + virtual ~IRequestProcessor() = default; +}; + +class TSimpleRequest: public IRequestProcessor { +private: + YDB_ACCESSOR(double, ScopeWeight, 1); + const ESpecialTaskCategory Category; + const TString ScopeId; + const ui64 ProcessId; + TAtomicCounter Counter; + TAtomicCounter CounterTasks; + virtual TString DoDebugString() const override { + return TStringBuilder() << Counter.Val(); + } + + virtual void DoInitialize(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) override { + actorSystem.Send(distributorId, new TEvExecution::TEvRegisterProcess(TCPULimitsConfig(1000, ScopeWeight), Category, ScopeId, ProcessId)); + } + virtual void DoAddTask(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) override { + actorSystem.Send(distributorId, + new TEvExecution::TEvNewTask(std::make_shared(TDuration::MicroSeconds(40), Counter), Category, ScopeId, ProcessId)); + CounterTasks.Inc(); + } + virtual bool DoCheckFinished() override { + return CounterTasks.Val() == Counter.Val(); + } + virtual void DoFinish(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId, const TDuration /*d*/) override { + actorSystem.Send(distributorId, new TEvExecution::TEvUnregisterProcess(Category, ScopeId, ProcessId)); + } + +public: + TSimpleRequest(const TString& id, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + : IRequestProcessor(id) + , Category(category) + , ScopeId(scopeId) + , ProcessId(processId) { + } +}; + +class TTestingExecutor { +private: + virtual TString GetConveyorConfig() = 0; + virtual std::vector> GetRequests() = 0; + virtual ui32 GetTasksCount() const { + return 1000000; + } +public: + virtual double GetThreadsCount() const { + return 9.5; + } + + void Execute() { + const ui64 threadsCount = 64; + THolder actorSystemSetup = NKikimr::BuildActorSystemSetup(threadsCount, 1); + NActors::TActorSystem actorSystem(actorSystemSetup); + + actorSystem.Start(); + auto counters = MakeIntrusive<::NMonitoring::TDynamicCounters>(); + const std::string textProto = GetConveyorConfig(); + NKikimrConfig::TCompositeConveyorConfig protoConfig; + AFL_VERIFY(google::protobuf::TextFormat::ParseFromString(textProto, &protoConfig)); + + NConfig::TConfig config; + config.DeserializeFromProto(protoConfig, threadsCount).Validate(); + const auto actorId = actorSystem.Register(TCompServiceOperator::CreateService(config, counters)); + + std::vector> requests = GetRequests(); + for (auto&& i : requests) { + i->Initialize(actorSystem, actorId); + } + for (ui32 i = 0; i < GetTasksCount(); ++i) { + for (auto&& i : requests) { + i->AddTask(actorSystem, actorId); + } + } + const TMonotonic globalStart = TMonotonic::Now(); + std::vector durations; + durations.resize(requests.size()); + { + bool isFinished = false; + while (!isFinished) { + isFinished = true; + ui32 idx = 0; + TStringBuilder sb; + for (auto&& i : requests) { + if (!i->CheckFinished()) { + isFinished = false; + } else if (!durations[idx]) { + durations[idx] = TMonotonic::Now() - globalStart; + } + sb << i->DebugString() << ";"; + ++idx; + } + Cerr << sb << Endl; + if (!isFinished) { + Sleep(TDuration::Seconds(1)); + } + } + } + { + ui32 idx = 0; + for (auto&& i : requests) { + i->Finish(actorSystem, actorId, durations[idx]); + ++idx; + } + } + Cerr << (GetThreadsCount() * (TMonotonic::Now() - globalStart) / (1.0 * requests.size() * GetTasksCount())).MicroSeconds() << "us per task" + << Endl; + TStringBuilder sb; + for (auto&& i : durations) { + sb << i << ";"; + } + Cerr << sb << Endl; + Sleep(TDuration::Seconds(5)); + + actorSystem.Stop(); + actorSystem.Cleanup(); + }; +}; + +Y_UNIT_TEST_SUITE(CompositeConveyorTests) { + class TTestingExecutor10xDistribution: public TTestingExecutor { + private: + virtual TString GetConveyorConfig() override { + return Sprintf(R"( + WorkerPools { + WorkersCount: %f + Links { + Category: "insert" + Weight: 0.1 + } + Links { + Category: "scan" + Weight: 0.01 + } + Links { + Category: "normalizer" + Weight: 0.001 + } + } + Categories { + Name: "insert" + } + Categories { + Name: "normalizer" + } + Categories { + Name: "scan" + } + )", + GetThreadsCount()); + } + virtual std::vector> GetRequests() override { + return { std::make_shared("I", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("S", ESpecialTaskCategory::Scan, "1", 1), + std::make_shared("N", ESpecialTaskCategory::Normalizer, "1", 1) }; + } + + public: + }; + Y_UNIT_TEST(Test10xDistribution) { + TTestingExecutor10xDistribution().Execute(); + } + + class TTestingExecutor10xMultiDistribution: public TTestingExecutor { + private: + virtual TString GetConveyorConfig() override { + return Sprintf(R"( + WorkerPools { + WorkersCount: %f + Links { + Category: "scan" + Weight: 1 + } + } + WorkerPools { + WorkersCount: %f + Links { + Category: "insert" + Weight: 0.1 + } + Links { + Category: "scan" + Weight: 0.01 + } + Links { + Category: "normalizer" + Weight: 0.001 + } + } + Categories { + Name: "insert" + } + Categories { + Name: "normalizer" + } + Categories { + Name: "scan" + } + )", + GetThreadsCount(), GetThreadsCount()); + } + virtual std::vector> GetRequests() override { + return { std::make_shared("I", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("S", ESpecialTaskCategory::Scan, "1", 1), + std::make_shared("N", ESpecialTaskCategory::Normalizer, "1", 1) }; + } + + public: + }; + Y_UNIT_TEST(Test10xMultiDistribution) { + TTestingExecutor10xMultiDistribution().Execute(); + } + + class TTestingExecutorUniformProcessDistribution: public TTestingExecutor { + private: + virtual TString GetConveyorConfig() override { + return Sprintf(R"( + WorkerPools { + WorkersCount: %f + Links { + Category: "insert" + Weight: 1 + } + } + Categories { + Name: "insert" + } + )", + GetThreadsCount()); + } + virtual std::vector> GetRequests() override { + return { std::make_shared("1", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("2", ESpecialTaskCategory::Insert, "1", 2), + std::make_shared("3", ESpecialTaskCategory::Insert, "1", 3) }; + } + + public: + }; + Y_UNIT_TEST(TestUniformProcessDistribution) { + TTestingExecutorUniformProcessDistribution().Execute(); + } + + class TTestingExecutorUniformScopesDistribution: public TTestingExecutor { + private: + virtual TString GetConveyorConfig() override { + return Sprintf(R"( + WorkerPools { + WorkersCount: %f + Links { + Category: "insert" + Weight: 1 + } + } + Categories { + Name: "insert" + } + )", + GetThreadsCount()); + } + virtual std::vector> GetRequests() override { + return { std::make_shared("1", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("2", ESpecialTaskCategory::Insert, "2", 1), + std::make_shared("3", ESpecialTaskCategory::Insert, "3", 1) }; + } + + public: + }; + Y_UNIT_TEST(TestUniformScopesDistribution) { + TTestingExecutorUniformScopesDistribution().Execute(); + } + + class TTestingExecutorUniformDistribution: public TTestingExecutor { + private: + virtual ui32 GetTasksCount() const override { + return 1000000; + } + virtual double GetThreadsCount() const override { + return 16.4; + } + virtual TString GetConveyorConfig() override { + return Sprintf(R"( + WorkerPools { + WorkersCount: %f + Links { + Category: "insert" + Weight: 0.1 + } + Links { + Category: "scan" + Weight: 0.1 + } + Links { + Category: "normalizer" + Weight: 0.1 + } + } + Categories { + Name: "insert" + } + Categories { + Name: "normalizer" + } + Categories { + Name: "scan" + } + )", + GetThreadsCount()); + } + virtual std::vector> GetRequests() override { + return { + std::make_shared("I_1_1", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("I_2_1", ESpecialTaskCategory::Insert, "2", 1), + std::make_shared("I_3_1", ESpecialTaskCategory::Insert, "3", 1), + std::make_shared("S_1_1", ESpecialTaskCategory::Scan, "1", 1), + std::make_shared("S_2_1", ESpecialTaskCategory::Scan, "2", 1), + std::make_shared("S_3_1", ESpecialTaskCategory::Scan, "3", 1), + std::make_shared("N_1_1", ESpecialTaskCategory::Normalizer, "1", 1), + std::make_shared("N_2_1", ESpecialTaskCategory::Normalizer, "2", 1), + std::make_shared("N_3_1", ESpecialTaskCategory::Normalizer, "3", 1), + std::make_shared("I_1_2", ESpecialTaskCategory::Insert, "1", 2), + std::make_shared("I_2_2", ESpecialTaskCategory::Insert, "2", 2), + std::make_shared("I_3_2", ESpecialTaskCategory::Insert, "3", 2), + std::make_shared("S_1_2", ESpecialTaskCategory::Scan, "1", 2), + std::make_shared("S_2_2", ESpecialTaskCategory::Scan, "2", 2), + std::make_shared("S_3_2", ESpecialTaskCategory::Scan, "3", 2), + std::make_shared("N_1_2", ESpecialTaskCategory::Normalizer, "1", 2), + std::make_shared("N_2_2", ESpecialTaskCategory::Normalizer, "2", 2), + std::make_shared("N_3_2", ESpecialTaskCategory::Normalizer, "3", 2) + }; + } + + public: + }; + Y_UNIT_TEST(TestUniformDistribution) { + TTestingExecutorUniformDistribution().Execute(); + } +} diff --git a/ydb/core/tx/conveyor_composite/ut/ya.make b/ydb/core/tx/conveyor_composite/ut/ya.make new file mode 100644 index 000000000000..9daa068c5045 --- /dev/null +++ b/ydb/core/tx/conveyor_composite/ut/ya.make @@ -0,0 +1,30 @@ +UNITTEST_FOR(ydb/core/tx/conveyor_composite/service) + +FORK_SUBTESTS() + +SPLIT_FACTOR(60) +SIZE(MEDIUM) + +PEERDIR( + contrib/libs/apache/arrow + ydb/core/base + ydb/core/tablet + ydb/core/tablet_flat + ydb/core/tx/columnshard/counters + yql/essentials/sql/pg_dummy + yql/essentials/core/arrow_kernels/request + ydb/core/testlib/default + ydb/core/tx/columnshard/test_helper + ydb/core/tx/columnshard/hooks/abstract + ydb/core/tx/columnshard/hooks/testing + + yql/essentials/udfs/common/json2 +) + +YQL_LAST_ABI_VERSION() + +SRCS( + ut_simple.cpp +) + +END() diff --git a/ydb/core/tx/conveyor_composite/ya.make b/ydb/core/tx/conveyor_composite/ya.make new file mode 100644 index 000000000000..efa8055bf69d --- /dev/null +++ b/ydb/core/tx/conveyor_composite/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( +) + +PEERDIR( + ydb/core/tx/conveyor_composite/service + ydb/core/tx/conveyor_composite/usage +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/tx/ya.make b/ydb/core/tx/ya.make index b3fcead3c1eb..544315c8e567 100644 --- a/ydb/core/tx/ya.make +++ b/ydb/core/tx/ya.make @@ -24,6 +24,8 @@ END() RECURSE( balance_coverage + conveyor + conveyor_composite columnshard coordinator datashard From dd973213f8cd756ab482fd08e6f1095539a6a087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9E=D0=BB=D0=B5=D0=B3?= <150132506+iddqdex@users.noreply.github.com> Date: Wed, 4 Jun 2025 19:59:06 +0300 Subject: [PATCH 06/62] Add RAM for test with sanitizers (#19326) --- ydb/core/tx/conveyor_composite/ut/ya.make | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/core/tx/conveyor_composite/ut/ya.make b/ydb/core/tx/conveyor_composite/ut/ya.make index 9daa068c5045..b85c559d0a72 100644 --- a/ydb/core/tx/conveyor_composite/ut/ya.make +++ b/ydb/core/tx/conveyor_composite/ut/ya.make @@ -5,6 +5,10 @@ FORK_SUBTESTS() SPLIT_FACTOR(60) SIZE(MEDIUM) +IF (SANITIZER_TYPE) + REQUIREMENTS(ram:16) +ENDIF() + PEERDIR( contrib/libs/apache/arrow ydb/core/base From a7d09cd7acb813e5b7d1903b77599bf0a449aca3 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Mon, 9 Jun 2025 21:27:58 +0300 Subject: [PATCH 07/62] composite conveyor usage (#19462) --- .../run/kikimr_services_initializers.cpp | 157 +++++++++++------- .../run/kikimr_services_initializers.h | 18 +- ydb/core/driver_lib/run/run.cpp | 12 +- ydb/core/kqp/query_data/kqp_predictor.cpp | 2 +- ydb/core/protos/config.proto | 1 + ydb/core/protos/console_config.proto | 2 + ydb/core/testlib/test_client.cpp | 11 +- .../tx/columnshard/columnshard__write.cpp | 4 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 8 +- .../engines/reader/abstract/read_context.cpp | 6 +- .../engines/reader/abstract/read_context.h | 7 +- .../engines/reader/actor/actor.cpp | 2 +- .../columnshard/engines/reader/actor/actor.h | 10 +- .../common_reader/iterator/constructor.cpp | 5 +- .../common_reader/iterator/fetch_steps.cpp | 6 +- .../reader/plain_reader/iterator/fetching.cpp | 4 +- .../reader/plain_reader/iterator/merge.cpp | 5 +- .../reader/plain_reader/iterator/source.cpp | 6 +- .../simple_reader/iterator/fetching.cpp | 8 +- .../reader/simple_reader/iterator/source.cpp | 8 +- .../engines/reader/transaction/tx_scan.cpp | 2 +- .../engines/writer/buffer/actor2.cpp | 3 +- .../normalizer/portion/normalizer.h | 4 +- .../operations/batch_builder/builder.cpp | 14 +- .../operations/batch_builder/restore.cpp | 22 +-- ydb/core/tx/columnshard/operations/write.cpp | 10 +- ydb/core/tx/columnshard/ya.make | 1 + ydb/core/tx/conveyor/usage/service.h | 14 -- .../conveyor_composite/service/category.cpp | 55 +++--- .../tx/conveyor_composite/service/category.h | 59 +++++-- .../tx/conveyor_composite/service/common.cpp | 4 - .../tx/conveyor_composite/service/common.h | 15 -- .../conveyor_composite/service/counters.cpp | 28 ++-- .../tx/conveyor_composite/service/counters.h | 84 +++++++--- .../tx/conveyor_composite/service/events.h | 47 +++++- .../tx/conveyor_composite/service/manager.h | 10 +- .../tx/conveyor_composite/service/process.h | 44 +++-- .../tx/conveyor_composite/service/scope.cpp | 33 ---- .../tx/conveyor_composite/service/scope.h | 71 ++++---- .../tx/conveyor_composite/service/service.cpp | 46 ++--- .../tx/conveyor_composite/service/service.h | 2 +- .../tx/conveyor_composite/service/worker.cpp | 2 - .../tx/conveyor_composite/service/worker.h | 13 +- .../service/workers_pool.cpp | 60 +++++-- .../conveyor_composite/service/workers_pool.h | 29 ++-- .../tx/conveyor_composite/usage/common.cpp | 15 +- ydb/core/tx/conveyor_composite/usage/common.h | 18 +- .../tx/conveyor_composite/usage/config.cpp | 139 ++++++++++++---- ydb/core/tx/conveyor_composite/usage/config.h | 56 +++++-- .../tx/conveyor_composite/usage/events.cpp | 5 +- ydb/core/tx/conveyor_composite/usage/events.h | 21 +-- .../tx/conveyor_composite/usage/service.h | 58 ++++--- .../tx/conveyor_composite/ut/ut_simple.cpp | 60 +++---- ydb/library/signals/owner.cpp | 4 +- 54 files changed, 782 insertions(+), 548 deletions(-) diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index ceeeb8665d6f..d981d5122326 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -186,6 +186,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -2215,75 +2218,111 @@ void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup* } } -TCompConveyorInitializer::TCompConveyorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} - -void TCompConveyorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - NConveyor::TConfig serviceConfig; - if (Config.HasCompConveyorConfig()) { - Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetCompConveyorConfig())); - } - if (!serviceConfig.HasDefaultFractionOfThreadsCount()) { - serviceConfig.SetDefaultFractionOfThreadsCount(0.33); - } - - if (serviceConfig.IsEnabled()) { - TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); - TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_COMP_CONVEYOR"); - - auto service = NConveyor::TCompServiceOperator::CreateService(serviceConfig, conveyorGroup); - - setup->LocalServices.push_back(std::make_pair( - NConveyor::TCompServiceOperator::MakeServiceId(NodeId), - TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); - } -} - -TScanConveyorInitializer::TScanConveyorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { +TCompositeConveyorInitializer::TCompositeConveyorInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { } -void TScanConveyorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - NConveyor::TConfig serviceConfig; - if (Config.HasScanConveyorConfig()) { - Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetScanConveyorConfig())); - } - - if (serviceConfig.IsEnabled()) { - TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); - TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_SCAN_CONVEYOR"); - - auto service = NConveyor::TScanServiceOperator::CreateService(serviceConfig, conveyorGroup); - - setup->LocalServices.push_back(std::make_pair( - NConveyor::TScanServiceOperator::MakeServiceId(NodeId), - TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); - } -} +void TCompositeConveyorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + const NKikimrConfig::TCompositeConveyorConfig protoConfig = [&]() { + if (Config.HasCompositeConveyorConfig()) { + return Config.GetCompositeConveyorConfig(); + } + NKikimrConfig::TCompositeConveyorConfig result; + if (Config.HasCompConveyorConfig()) { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Compaction)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Compaction)); + protoLink.SetWeight(1); + if (Config.GetCompConveyorConfig().HasWorkersCount()) { + protoWorkersPool.SetWorkersCount(Config.GetCompConveyorConfig().GetWorkersCount()); + } else if (Config.GetCompConveyorConfig().HasWorkersCountDouble()) { + protoWorkersPool.SetWorkersCount(Config.GetCompConveyorConfig().GetWorkersCountDouble()); + } else if (Config.GetCompConveyorConfig().HasDefaultFractionOfThreadsCount()) { + protoWorkersPool.SetDefaultFractionOfThreadsCount(Config.GetCompConveyorConfig().GetDefaultFractionOfThreadsCount()); + } else { + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.33); + } + } else { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Compaction)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Compaction)); + protoLink.SetWeight(1); + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.33); + } -TInsertConveyorInitializer::TInsertConveyorInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) { -} + if (Config.HasInsertConveyorConfig()) { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Insert)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Insert)); + protoLink.SetWeight(1); + if (Config.GetInsertConveyorConfig().HasWorkersCount()) { + protoWorkersPool.SetWorkersCount(Config.GetInsertConveyorConfig().GetWorkersCount()); + } else if (Config.GetInsertConveyorConfig().HasWorkersCountDouble()) { + protoWorkersPool.SetWorkersCount(Config.GetInsertConveyorConfig().GetWorkersCountDouble()); + } else if (Config.GetCompConveyorConfig().HasDefaultFractionOfThreadsCount()) { + protoWorkersPool.SetDefaultFractionOfThreadsCount(Config.GetCompConveyorConfig().GetDefaultFractionOfThreadsCount()); + } else { + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.2); + } + } else { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Insert)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Insert)); + protoLink.SetWeight(1); + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.2); + } + if (Config.HasScanConveyorConfig()) { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Scan)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Scan)); + protoLink.SetWeight(1); + if (Config.GetScanConveyorConfig().HasWorkersCount()) { + protoWorkersPool.SetWorkersCount(Config.GetScanConveyorConfig().GetWorkersCount()); + } else if (Config.GetScanConveyorConfig().HasWorkersCountDouble()) { + protoWorkersPool.SetWorkersCount(Config.GetScanConveyorConfig().GetWorkersCountDouble()); + } else if (Config.GetCompConveyorConfig().HasDefaultFractionOfThreadsCount()) { + protoWorkersPool.SetDefaultFractionOfThreadsCount(Config.GetCompConveyorConfig().GetDefaultFractionOfThreadsCount()); + } else { + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.4); + } + } else { + NKikimrConfig::TCompositeConveyorConfig::TCategory& protoCategory = *result.AddCategories(); + protoCategory.SetName(::ToString(NConveyorComposite::ESpecialTaskCategory::Scan)); + NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& protoWorkersPool = *result.AddWorkerPools(); + NKikimrConfig::TCompositeConveyorConfig::TWorkerPoolCategoryLink& protoLink = *protoWorkersPool.AddLinks(); + protoLink.SetCategory(::ToString(NConveyorComposite::ESpecialTaskCategory::Scan)); + protoLink.SetWeight(1); + protoWorkersPool.SetDefaultFractionOfThreadsCount(0.4); + } + return result; + }(); -void TInsertConveyorInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { - NConveyor::TConfig serviceConfig; - if (Config.HasInsertConveyorConfig()) { - Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetInsertConveyorConfig())); - } - if (!serviceConfig.HasDefaultFractionOfThreadsCount()) { - serviceConfig.SetDefaultFractionOfThreadsCount(0.2); + auto serviceConfig = NConveyorComposite::NConfig::TConfig::BuildFromProto(protoConfig); + if (serviceConfig.IsFail()) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse composite conveyor config")("action", "default_usage")( + "error", serviceConfig.GetErrorMessage())("default", NConveyorComposite::NConfig::TConfig::BuildDefault().DebugString()); + serviceConfig = NConveyorComposite::NConfig::TConfig::BuildDefault(); } + AFL_VERIFY(!serviceConfig.IsFail()); - if (serviceConfig.IsEnabled()) { + if (serviceConfig->IsEnabled()) { TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); - TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_INSERT_CONVEYOR"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_COMPOSITE_CONVEYOR"); - auto service = NConveyor::TInsertServiceOperator::CreateService(serviceConfig, conveyorGroup); + auto service = NConveyorComposite::TServiceOperator::CreateService(*serviceConfig, conveyorGroup); setup->LocalServices.push_back(std::make_pair( - NConveyor::TInsertServiceOperator::MakeServiceId(NodeId), - TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); + NConveyorComposite::TServiceOperator::MakeServiceId(NodeId), TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); } } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index cadb077a875e..0d41ea7b8cae 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -416,22 +416,10 @@ class TCompPrioritiesInitializer: public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TCompConveyorInitializer: public IKikimrServicesInitializer { +class TCompositeConveyorInitializer : public IKikimrServicesInitializer { public: - TCompConveyorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - -class TScanConveyorInitializer: public IKikimrServicesInitializer { -public: - TScanConveyorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; -}; - -class TInsertConveyorInitializer: public IKikimrServicesInitializer { -public: - TInsertConveyorInitializer(const TKikimrRunConfig& runConfig); - void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; + TCompositeConveyorInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; class TExternalIndexInitializer: public IKikimrServicesInitializer { diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 43a6800d4a1b..0c1e016f6478 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1703,20 +1703,12 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TGroupedMemoryLimiterInitializer(runConfig)); } - if (serviceMask.EnableScanConveyor) { - sil->AddServiceInitializer(new TScanConveyorInitializer(runConfig)); - } - if (serviceMask.EnableCompPriorities) { sil->AddServiceInitializer(new TCompPrioritiesInitializer(runConfig)); } - if (serviceMask.EnableCompConveyor) { - sil->AddServiceInitializer(new TCompConveyorInitializer(runConfig)); - } - - if (serviceMask.EnableInsertConveyor) { - sil->AddServiceInitializer(new TInsertConveyorInitializer(runConfig)); + if (serviceMask.EnableCompConveyor || serviceMask.EnableInsertConveyor || serviceMask.EnableScanConveyor) { + sil->AddServiceInitializer(new TCompositeConveyorInitializer(runConfig)); } if (serviceMask.EnableCms) { diff --git a/ydb/core/kqp/query_data/kqp_predictor.cpp b/ydb/core/kqp/query_data/kqp_predictor.cpp index abd5f914f29e..5380c89978f9 100644 --- a/ydb/core/kqp/query_data/kqp_predictor.cpp +++ b/ydb/core/kqp/query_data/kqp_predictor.cpp @@ -127,7 +127,7 @@ bool TStagePredictor::DeserializeFromKqpSettings(const NYql::NDqProto::TProgram: ui32 TStagePredictor::GetUsableThreads() { std::optional userPoolSize; - if (TlsActivationContext && TlsActivationContext->ActorSystem()) { + if (HasAppData() && TlsActivationContext && TlsActivationContext->ActorSystem()) { userPoolSize = TlsActivationContext->ActorSystem()->GetPoolThreadsCount(AppData()->UserPoolId); } if (!userPoolSize) { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 523dc17698fa..8943984e1d5d 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -652,6 +652,7 @@ message TCompositeConveyorConfig { optional double WorkersCount = 1; optional double DefaultFractionOfThreadsCount = 2; repeated TWorkerPoolCategoryLink Links = 3; + optional string Name = 4; } optional bool Enabled = 1 [default = true]; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index a1fae3eb738d..6372aee5f6c7 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -148,6 +148,8 @@ message TConfigItem { NamedConfigsItem = 100; ClusterYamlConfigItem = 101; + CompositeConveyorConfigItem = 109; + // synthetic kinds for audit purposes only DatabaseYamlConfigChangeItem = 32767; MainYamlConfigChangeItem = 32768; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c6257c1227c8..27c563af5504 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -125,6 +125,8 @@ #include #include #include +#include +#include #include #include #include @@ -1130,14 +1132,9 @@ namespace Tests { Runtime->RegisterService(NConveyor::TScanServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } { - auto* actor = NConveyor::TCompServiceOperator::CreateService(NConveyor::TConfig(), new ::NMonitoring::TDynamicCounters()); + auto* actor = NConveyorComposite::TServiceOperator::CreateService(NConveyorComposite::NConfig::TConfig::BuildDefault(), new ::NMonitoring::TDynamicCounters()); const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); - Runtime->RegisterService(NConveyor::TCompServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); - } - { - auto* actor = NConveyor::TInsertServiceOperator::CreateService(NConveyor::TConfig(), new ::NMonitoring::TDynamicCounters()); - const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); - Runtime->RegisterService(NConveyor::TInsertServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); + Runtime->RegisterService(NConveyorComposite::TServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0); diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index b3144f6dd4cc..1c7960226e2c 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -14,7 +14,7 @@ #include "transactions/operators/ev_write/sync.h" #include -#include +#include #include namespace NKikimr::NColumnShard { @@ -297,7 +297,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex Counters.GetCSCounters().WritingCounters, GetLastTxSnapshot(), std::make_shared(1), true, BufferizationInsertionWriteActorId, BufferizationPortionsWriteActorId); std::shared_ptr task = std::make_shared(std::move(writeData), context); - NConveyor::TInsertServiceOperator::AsyncTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); } } diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index de762d37fc2c..3e4ec1b62678 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -596,9 +597,9 @@ class TChangesReadTask: public NOlap::NBlobOperations::NRead::ITask { std::shared_ptr task = std::make_shared(std::move(TxEvent), Counters, TabletId, ParentActorId, LastCompletedTx); if (isInsert) { - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); } else { - NConveyor::TCompServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TCompServiceOperator::SendTaskToExecute(task); } } virtual bool DoOnError(const TString& storageId, const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { @@ -1510,7 +1511,8 @@ class TTxAskPortionChunks: public TTransactionBase { } AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("stage", "finished"); - NConveyor::TInsertServiceOperator::AsyncTaskToExecute(std::make_shared(FetchCallback, std::move(FetchedAccessors))); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute( + std::make_shared(FetchCallback, std::move(FetchedAccessors)), 0); return true; } void Complete(const TActorContext& /*ctx*/) override { diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.cpp b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.cpp index 261c59416624..3fca2962fdef 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.cpp @@ -1,7 +1,7 @@ #include "read_context.h" #include -#include +#include namespace NKikimr::NOlap::NReader { @@ -13,7 +13,7 @@ TReadContext::TReadContext(const std::shared_ptr& storagesMana const std::shared_ptr& dataAccessorsManager, const NColumnShard::TConcreteScanCounters& counters, const TReadMetadataBase::TConstPtr& readMetadata, const TActorId& scanActorId, const TActorId& resourceSubscribeActorId, const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy, - const ui64 scanId, const NConveyor::TCPULimitsConfig& cpuLimits) + const ui64 scanId, const NConveyorComposite::TCPULimitsConfig& cpuLimits) : StoragesManager(storagesManager) , DataAccessorsManager(dataAccessorsManager) , Counters(counters) @@ -24,7 +24,7 @@ TReadContext::TReadContext(const std::shared_ptr& storagesMana , ResourceSubscribeActorId(resourceSubscribeActorId) , ReadCoordinatorActorId(readCoordinatorActorId) , ComputeShardingPolicy(computeShardingPolicy) - , ConveyorProcessGuard(NConveyor::TScanServiceOperator::StartProcess(ScanId, cpuLimits)) { + , ConveyorProcessGuard(NConveyorComposite::TScanServiceOperator::StartProcess(ScanId, ::ToString(ScanId), cpuLimits)) { Y_ABORT_UNLESS(ReadMetadata); if (ReadMetadata->HasResultSchema()) { Resolver = std::make_shared(ReadMetadata->GetResultSchema()->GetIndexInfo()); diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h index b8bff05dbb3a..a40d915e04fb 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -58,7 +59,7 @@ class TReadContext { const TComputeShardingPolicy ComputeShardingPolicy; std::shared_ptr AbortionFlag = std::make_shared(0); std::shared_ptr ConstAbortionFlag = AbortionFlag; - const NConveyor::TProcessGuard ConveyorProcessGuard; + const NConveyorComposite::TProcessGuard ConveyorProcessGuard; std::shared_ptr Resolver; public: @@ -68,7 +69,7 @@ class TReadContext { } ui64 GetConveyorProcessId() const { - return ConveyorProcessGuard.GetProcessId(); + return ConveyorProcessGuard.GetInternalProcessId(); } template @@ -149,7 +150,7 @@ class TReadContext { const std::shared_ptr& dataAccessorsManager, const NColumnShard::TConcreteScanCounters& counters, const TReadMetadataBase::TConstPtr& readMetadata, const TActorId& scanActorId, const TActorId& resourceSubscribeActorId, const TActorId& readCoordinatorActorId, const TComputeShardingPolicy& computeShardingPolicy, - const ui64 scanId, const NConveyor::TCPULimitsConfig& cpuLimits); + const ui64 scanId, const NConveyorComposite::TCPULimitsConfig& cpuLimits); }; class IDataReader { diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index 0be93b7ce792..f6de383c6548 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -21,7 +21,7 @@ TColumnShardScan::TColumnShardScan(const TActorId& columnShardActorId, const TAc const std::shared_ptr& dataAccessorsManager, const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, ui64 tabletId, TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, NKikimrDataEvents::EDataFormat dataFormat, - const NColumnShard::TScanCounters& scanCountersPool, const NConveyor::TCPULimitsConfig& cpuLimits) + const NColumnShard::TScanCounters& scanCountersPool, const NConveyorComposite::TCPULimitsConfig& cpuLimits) : StoragesManager(storagesManager) , DataAccessorsManager(dataAccessorsManager) , ColumnShardActorId(columnShardActorId) diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.h b/ydb/core/tx/columnshard/engines/reader/actor/actor.h index c4b49e43dee1..f50dee364195 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.h +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -35,15 +35,15 @@ class TColumnShardScan: public TActorBootstrapped, const std::shared_ptr& dataAccessorsManager, const TComputeShardingPolicy& computeShardingPolicy, ui32 scanId, ui64 txId, ui32 scanGen, ui64 requestCookie, ui64 tabletId, TDuration timeout, const TReadMetadataBase::TConstPtr& readMetadataRange, NKikimrDataEvents::EDataFormat dataFormat, - const NColumnShard::TScanCounters& scanCountersPool, const NConveyor::TCPULimitsConfig& cpuLimits); + const NColumnShard::TScanCounters& scanCountersPool, const NConveyorComposite::TCPULimitsConfig& cpuLimits); void Bootstrap(const TActorContext& ctx); private: STATEFN(StateScan) { auto g = Stats->MakeGuard("processing"); - TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_SCAN) ("SelfId", SelfId())( - "TabletId", TabletId)("ScanId", ScanId)("TxId", TxId)("ScanGen", ScanGen)("task_identifier", ReadMetadataRange->GetScanIdentifier())); + TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_SCAN) ("SelfId", SelfId())("TabletId", + TabletId)("ScanId", ScanId)("TxId", TxId)("ScanGen", ScanGen)("task_identifier", ReadMetadataRange->GetScanIdentifier())); switch (ev->GetTypeRewrite()) { hFunc(NKqp::TEvKqpCompute::TEvScanDataAck, HandleScan); hFunc(NKqp::TEvKqpCompute::TEvScanPing, HandleScan); @@ -130,7 +130,7 @@ class TColumnShardScan: public TActorBootstrapped, const ui64 RequestCookie; const NKikimrDataEvents::EDataFormat DataFormat; const ui64 TabletId; - const NConveyor::TCPULimitsConfig CPULimits; + const NConveyorComposite::TCPULimitsConfig CPULimits; TReadMetadataBase::TConstPtr ReadMetadataRange; std::unique_ptr ScanIterator; diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.cpp index ca50a7544531..9840408ee3cf 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/constructor.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace NKikimr::NOlap::NReader::NCommon { @@ -11,7 +12,7 @@ void TBlobsFetcherTask::DoOnDataReady(const std::shared_ptrMutableStageData().AddBlobs(Source->DecodeBlobAddresses(ExtractBlobsData())); AFL_VERIFY(Step.Next()); auto task = std::make_shared(Source, std::move(Step), Context->GetCommonContext()->GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, Context->GetCommonContext()->GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, Context->GetCommonContext()->GetConveyorProcessId()); } bool TBlobsFetcherTask::DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) { @@ -58,7 +59,7 @@ void TColumnsFetcherTask::DoOnDataReady(const std::shared_ptrMutableStageData().AddFetcher(i.second); } auto task = std::make_shared(Source, std::move(Cursor), Source->GetContext()->GetCommonContext()->GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, Source->GetContext()->GetCommonContext()->GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, Source->GetContext()->GetCommonContext()->GetConveyorProcessId()); } else { FOR_DEBUG_LOG(NKikimrServices::COLUMNSHARD_SCAN_EVLOG, Source->AddEvent("cf_next")); std::shared_ptr nextReadTask = std::make_shared( diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.cpp index 7b5220aeba28..f6ca9b63cdd7 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetch_steps.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -58,7 +58,7 @@ bool TAllocateMemoryStep::TFetchingStepAllocation::DoOnAllocated(std::shared_ptr } FOR_DEBUG_LOG(NKikimrServices::COLUMNSHARD_SCAN_EVLOG, data->AddEvent("fmalloc")); auto task = std::make_shared(data, std::move(Step), data->GetContext()->GetCommonContext()->GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, data->GetContext()->GetCommonContext()->GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, data->GetContext()->GetCommonContext()->GetConveyorProcessId()); return true; } @@ -86,7 +86,7 @@ TConclusion TAllocateMemoryStep::DoExecuteInplace(const std::shared_ptrGetColumnsVolume(i.GetColumns().GetColumnIds(), i.GetMemType()); - if (source->GetStageData().GetUseFilter() && i.GetMemType() != EMemType::Blob && source->GetContext()->GetReadMetadata()->HasLimit() && + if (source->GetStageData().GetUseFilter() && i.GetMemType() != EMemType::Blob && source->GetContext()->GetReadMetadata()->HasLimit() && (HasAppData() && !AppDataVerified().ColumnShardConfig.GetUseSlicesFilter())) { const ui32 filtered = source->GetStageData().GetFilteredCount(source->GetRecordsCount(), source->GetContext()->GetReadMetadata()->GetLimitRobust()); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp index 468ff7ed3c57..277d504da366 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include @@ -90,7 +90,7 @@ TConclusion TDetectInMem::DoExecuteInplace(const std::shared_ptrGetContext()->GetCommonContext(); auto task = std::make_shared(source, std::move(cursor), commonContext.GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); return false; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp index 411301cb67ac..ff653a984a63 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/merge.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace NKikimr::NOlap::NReader::NPlain { @@ -86,7 +86,8 @@ bool TBaseMergeTask::DoOnAllocated( return false; } AllocationGuard = std::move(guard); - NConveyor::TScanServiceOperator::SendTaskToExecute(static_pointer_cast(allocation), Context->GetCommonContext()->GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute( + static_pointer_cast(allocation), Context->GetCommonContext()->GetConveyorProcessId()); return true; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index fb2381816c34..0aa21c4d7f55 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include @@ -39,7 +39,7 @@ void IDataSource::RegisterInterval(TFetchingInterval& interval, const std::share TFetchingScriptCursor cursor(FetchingPlan, 0); const auto& commonContext = *GetContext()->GetCommonContext(); auto task = std::make_shared(sourcePtr, std::move(cursor), commonContext.GetScanActorId(), true); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); } } @@ -175,7 +175,7 @@ class TPortionAccessorFetchingSubscriber: public IDataAccessorRequestsSubscriber AFL_VERIFY(Step.Next()); const auto& commonContext = *Source->GetContext()->GetCommonContext(); auto task = std::make_shared(Source, std::move(Step), commonContext.GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); } public: diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp index 1728fea5f647..7ab73deeed57 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/fetching.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include @@ -103,7 +103,7 @@ TConclusion TDetectInMem::DoExecuteInplace(const std::shared_ptrAddEvent("sdmem")); const auto& commonContext = *source->GetContext()->GetCommonContext(); auto task = std::make_shared(source, std::move(cursor), commonContext.GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); return false; } @@ -186,7 +186,7 @@ TConclusion TPrepareResultStep::DoExecuteInplace(const std::shared_ptrGetCommonContext(); auto task = std::make_shared(source, std::move(cursor), commonContext.GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); return false; } else { return true; @@ -205,7 +205,7 @@ void TDuplicateFilter::TFilterSubscriber::OnFilterReady(NArrow::TColumnFilter&& source->MutableStageData().AddFilter(std::move(filter)); Step.Next(); auto task = std::make_shared(source, std::move(Step), source->GetContext()->GetCommonContext()->GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, source->GetContext()->GetCommonContext()->GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, source->GetContext()->GetCommonContext()->GetConveyorProcessId()); } } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index 7bae7dbafe53..43f0117d65e6 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -42,7 +42,7 @@ void IDataSource::StartProcessing(const std::shared_ptr& sourcePtr) TFetchingScriptCursor cursor(FetchingPlan, 0); const auto& commonContext = *GetContext()->GetCommonContext(); auto task = std::make_shared(sourcePtr, std::move(cursor), commonContext.GetScanActorId(), true); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); } void IDataSource::ContinueCursor(const std::shared_ptr& sourcePtr) { @@ -53,7 +53,7 @@ void IDataSource::ContinueCursor(const std::shared_ptr& sourcePtr) ScriptCursor.reset(); const auto& commonContext = *GetContext()->GetCommonContext(); auto task = std::make_shared(sourcePtr, std::move(cursor), commonContext.GetScanActorId(), true); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); } else { AFL_WARN(NKikimrServices::TX_COLUMNSHARD_SCAN)("source_id", GetSourceId())("event", "CannotContinueCursor"); } @@ -392,7 +392,7 @@ class TPortionAccessorFetchingSubscriber: public IDataAccessorRequestsSubscriber AFL_VERIFY(Step.Next()); const auto& commonContext = *Source->GetContext()->GetCommonContext(); auto task = std::make_shared(Source, std::move(Step), commonContext.GetScanActorId(), false); - NConveyor::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); + NConveyorComposite::TScanServiceOperator::SendTaskToExecute(task, commonContext.GetConveyorProcessId()); } public: diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index c5df7420f2c2..9c2865b556ff 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -54,7 +54,7 @@ void TTxScan::Complete(const TActorContext& ctx) { const TString table = request.GetTablePath(); const auto dataFormat = request.GetDataFormat(); const TDuration timeout = TDuration::MilliSeconds(request.GetTimeoutMs()); - NConveyor::TCPULimitsConfig cpuLimits; + NConveyorComposite::TCPULimitsConfig cpuLimits; cpuLimits.DeserializeFromProto(request).Validate(); if (scanGen > 1) { Self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_SCAN_RESTARTED); diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp index 329efccbd77a..b6b5ff586b47 100644 --- a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp +++ b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace NKikimr::NOlap::NWritingPortions { @@ -71,7 +72,7 @@ void TWriteAggregation::Flush(const ui64 tabletId) { Context.GetWritingCounters()->OnAggregationWrite(Units.size(), SumSize); std::shared_ptr task = std::make_shared(std::move(Units), Context, PathId, tabletId, ModificationType); - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); Units.clear(); SumSize = 0; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h index edfdad23ff4c..884f13a5d1a0 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.h +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace NKikimr::NColumnShard { class TTablesManager; @@ -36,7 +37,7 @@ class TReadPortionsTask: public NOlap::NBlobOperations::NRead::ITask { NormContext.SetResourcesGuard(resourcesGuard); std::shared_ptr task = std::make_shared(std::move(ExtractBlobsData()), NormContext, std::move(Data), Schemas); - NConveyor::TCompServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TNormalizerServiceOperator::SendTaskToExecute(task); } virtual bool DoOnError(const TString& storageId, const TBlobRange& range, const IBlobsReadingAction::TErrorStatus& status) override { @@ -83,6 +84,7 @@ class TPortionsNormalizerTask: public INormalizerTask { class TPortionsNormalizerBase: public TNormalizationController::INormalizerComponent { private: using TBase = TNormalizationController::INormalizerComponent; + public: TPortionsNormalizerBase(const TNormalizationController::TInitContext& info) : TBase(info) diff --git a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp index 7f886c8be62e..f501fb17535e 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace NKikimr::NOlap { @@ -49,12 +50,13 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { std::shared_ptr merger; switch (WriteData.GetWriteMeta().GetModificationType()) { case NEvWrite::EModificationType::Upsert: { - const std::vector> defaultFields = Context.GetActualSchema()->GetAbsentFields(batch.GetContainer()->schema()); + const std::vector> defaultFields = + Context.GetActualSchema()->GetAbsentFields(batch.GetContainer()->schema()); if (defaultFields.empty()) { if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batch.GetContainer(), Context); - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); } else { NActors::TActivationContext::ActorSystem()->Send(Context.GetBufferizationPortionsActorId(), new NWritingPortions::TEvAddInsertedDataToBuffer( @@ -68,8 +70,8 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { auto batchDefault = conclusion.DetachResult(); NArrow::NMerger::TSortableBatchPosition pos( batchDefault, 0, batchDefault->schema()->field_names(), batchDefault->schema()->field_names(), false); - merger = std::make_shared(batch, Context.GetActualSchema(), - insertionConclusion.IsSuccess() ? "" : insertionConclusion.GetErrorMessage(), pos); + merger = std::make_shared( + batch, Context.GetActualSchema(), insertionConclusion.IsSuccess() ? "" : insertionConclusion.GetErrorMessage(), pos); break; } } @@ -86,11 +88,11 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batch.GetContainer(), Context); - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); } else { NActors::TActivationContext::ActorSystem()->Send(Context.GetBufferizationPortionsActorId(), new NWritingPortions::TEvAddInsertedDataToBuffer( - std::make_shared(WriteData), batch, std::make_shared(Context))); + std::make_shared(WriteData), batch, std::make_shared(Context))); } return; } diff --git a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp index 770258d038e7..049c6c100b2a 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp @@ -4,14 +4,15 @@ #include #include #include +#include namespace NKikimr::NOlap { std::unique_ptr TModificationRestoreTask::DoBuildRequestInitiator() const { auto request = std::make_unique(LocalPathId, Snapshot, WriteData.GetWriteMeta().GetLockIdOptional()); request->TaskIdentifier = GetTaskId(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_RESTORE)("event", "restore_start")("count", IncomingData.HasContainer() ? IncomingData->num_rows() : 0)( - "task_id", WriteData.GetWriteMeta().GetId()); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_RESTORE)("event", "restore_start")( + "count", IncomingData.HasContainer() ? IncomingData->num_rows() : 0)("task_id", WriteData.GetWriteMeta().GetId()); auto pkData = NArrow::TColumnOperator().VerifyIfAbsent().Extract(IncomingData.GetContainer(), Context.GetActualSchema()->GetPKColumnNames()); request->RangesFilter = TPKRangesFilter::BuildFromRecordBatchLines(pkData); for (auto&& i : Context.GetActualSchema()->GetIndexInfo().GetColumnIds(false)) { @@ -25,7 +26,9 @@ TConclusionStatus TModificationRestoreTask::DoOnDataChunk(const std::shared_ptr< if (result.IsFail()) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD_RESTORE)("event", "merge_data_problems")("write_id", WriteData.GetWriteMeta().GetWriteId())( "tablet_id", GetTabletId())("message", result.GetErrorMessage()); - SendErrorMessage(result.GetErrorMessage(), result.GetStatus() == Ydb::StatusIds::PRECONDITION_FAILED ? NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::ConstraintViolation : NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Request); + SendErrorMessage(result.GetErrorMessage(), result.GetStatus() == Ydb::StatusIds::PRECONDITION_FAILED + ? NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::ConstraintViolation + : NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Request); return TConclusionStatus::Fail(result.GetErrorMessage()); } return TConclusionStatus::Success(); @@ -48,18 +51,17 @@ NKikimr::TConclusionStatus TModificationRestoreTask::DoOnFinished() { if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batchResult.GetContainer(), Context); - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); } else { - NActors::TActivationContext::ActorSystem()->Send( - Context.GetBufferizationPortionsActorId(), new NWritingPortions::TEvAddInsertedDataToBuffer( - std::make_shared(WriteData), batchResult, std::make_shared(Context))); + NActors::TActivationContext::ActorSystem()->Send(Context.GetBufferizationPortionsActorId(), + new NWritingPortions::TEvAddInsertedDataToBuffer( + std::make_shared(WriteData), batchResult, std::make_shared(Context))); } return TConclusionStatus::Success(); } TModificationRestoreTask::TModificationRestoreTask(NEvWrite::TWriteData&& writeData, const std::shared_ptr& merger, - const TSnapshot actualSnapshot, const NArrow::TContainerWithIndexes& incomingData, - const TWritingContext& context) + const TSnapshot actualSnapshot, const NArrow::TContainerWithIndexes& incomingData, const TWritingContext& context) : TBase(context.GetTabletId(), context.GetTabletActorId(), writeData.GetWriteMeta().GetId() + "::" + ::ToString(writeData.GetWriteMeta().GetWriteId())) , WriteData(std::move(writeData)) @@ -68,7 +70,7 @@ TModificationRestoreTask::TModificationRestoreTask(NEvWrite::TWriteData&& writeD , Snapshot(actualSnapshot) , IncomingData(incomingData) , Context(context) { - AFL_VERIFY(actualSnapshot.Valid()); + AFL_VERIFY(actualSnapshot.Valid()); } void TModificationRestoreTask::SendErrorMessage( diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index b7d184b9c07c..68260095e2f4 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace NKikimr::NColumnShard { @@ -31,14 +32,14 @@ void TWriteOperation::Start( TColumnShard& owner, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const NOlap::TWritingContext& context) { Y_ABORT_UNLESS(Status == EOperationStatus::Draft); - auto writeMeta = std::make_shared((ui64)WriteId, GetPathId(), source, GranuleShardingVersionId, GetIdentifier(), - context.GetWritingCounters()->GetWriteFlowCounters()); + auto writeMeta = std::make_shared( + (ui64)WriteId, GetPathId(), source, GranuleShardingVersionId, GetIdentifier(), context.GetWritingCounters()->GetWriteFlowCounters()); writeMeta->SetLockId(LockId); writeMeta->SetModificationType(ModificationType); NEvWrite::TWriteData writeData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR), WritePortions); std::shared_ptr task = std::make_shared(std::move(writeData), context); - NConveyor::TInsertServiceOperator::SendTaskToExecute(task); + NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); Status = EOperationStatus::Started; } @@ -138,7 +139,8 @@ void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::T owner.InsertTable->Abort(dbTable, writeIds); } else { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnExecute(txc, i, owner.GetCurrentSnapshotForInternalModification()); + owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnExecute( + txc, i, owner.GetCurrentSnapshotForInternalModification()); } } } diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index ffd0563fc026..e2eaf6279c25 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -63,6 +63,7 @@ PEERDIR( ydb/core/tx/columnshard/transactions/operators ydb/core/tx/columnshard/tx_reader ydb/core/tx/conveyor/usage + ydb/core/tx/conveyor_composite/service ydb/core/tx/long_tx_service/public ydb/core/tx/priorities/service ydb/core/tx/tiering diff --git a/ydb/core/tx/conveyor/usage/service.h b/ydb/core/tx/conveyor/usage/service.h index a943ec29b62a..eac7e562e647 100644 --- a/ydb/core/tx/conveyor/usage/service.h +++ b/ydb/core/tx/conveyor/usage/service.h @@ -90,20 +90,6 @@ class TScanConveyorPolicy { static constexpr bool EnableProcesses = true; }; -class TCompConveyorPolicy { -public: - static const inline TString Name = "Comp"; - static constexpr bool EnableProcesses = false; -}; - -class TInsertConveyorPolicy { -public: - static const inline TString Name = "Isrt"; - static constexpr bool EnableProcesses = false; -}; - using TScanServiceOperator = TServiceOperatorImpl; -using TCompServiceOperator = TServiceOperatorImpl; -using TInsertServiceOperator = TServiceOperatorImpl; } // namespace NKikimr::NConveyor diff --git a/ydb/core/tx/conveyor_composite/service/category.cpp b/ydb/core/tx/conveyor_composite/service/category.cpp index 4c87ae85d0d8..0f2234794e45 100644 --- a/ydb/core/tx/conveyor_composite/service/category.cpp +++ b/ydb/core/tx/conveyor_composite/service/category.cpp @@ -3,36 +3,36 @@ namespace NKikimr::NConveyorComposite { bool TProcessCategory::HasTasks() const { - for (auto&& i : Scopes) { - if (i.second->HasTasks()) { - return true; - } - } - return false; + return ProcessesWithTasks.size(); } void TProcessCategory::DoQuant(const TMonotonic newStart) { CPUUsage->Cut(newStart); - for (auto&& i : Scopes) { + for (auto&& i : Processes) { i.second->DoQuant(newStart); } } -TWorkerTask TProcessCategory::ExtractTaskWithPrediction() { - std::shared_ptr scopeMin; +TWorkerTask TProcessCategory::ExtractTaskWithPrediction(const std::shared_ptr& counters) { + std::shared_ptr pMin; TDuration dMin; - for (auto&& [_, scope] : Scopes) { - if (!scope->HasTasks()) { + for (auto&& [_, p] : ProcessesWithTasks) { + if (!p->GetScope()->CheckToRun()) { continue; } - const TDuration d = scope->GetCPUUsage()->CalcWeight(scope->GetWeight()); - if (!scopeMin || d < dMin) { + const TDuration d = p->GetCPUUsage()->CalcWeight(p->GetWeight()); + if (!pMin || d < dMin) { dMin = d; - scopeMin = scope; + pMin = p; } } - AFL_VERIFY(scopeMin); - return scopeMin->ExtractTaskWithPrediction(); + AFL_VERIFY(pMin); + auto result = pMin->ExtractTaskWithPrediction(counters); + if (pMin->GetTasksCount() == 0) { + AFL_VERIFY(ProcessesWithTasks.erase(pMin->GetProcessId())); + } + Counters->WaitingQueueSize->Set(WaitingTasksCount->Val()); + return result; } TProcessScope& TProcessCategory::MutableProcessScope(const TString& scopeName) { @@ -41,6 +41,12 @@ TProcessScope& TProcessCategory::MutableProcessScope(const TString& scopeName) { return *it->second; } +std::shared_ptr TProcessCategory::GetProcessScopePtrVerified(const TString& scopeName) const { + auto it = Scopes.find(scopeName); + AFL_VERIFY(it != Scopes.end()); + return it->second; +} + TProcessScope* TProcessCategory::MutableProcessScopeOptional(const TString& scopeName) { auto it = Scopes.find(scopeName); if (it != Scopes.end()) { @@ -50,20 +56,20 @@ TProcessScope* TProcessCategory::MutableProcessScopeOptional(const TString& scop } } -TProcessScope& TProcessCategory::RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { +std::shared_ptr TProcessCategory::RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { TCPUGroup::TPtr cpuGroup = std::make_shared(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); - auto info = Scopes.emplace(scopeId, std::make_shared(std::move(cpuGroup), CPUUsage)); + auto info = Scopes.emplace(scopeId, std::make_shared(scopeId, std::move(cpuGroup), std::move(CPUUsage))); AFL_VERIFY(info.second); - return *info.first->second; + return info.first->second; } -TProcessScope& TProcessCategory::UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { - auto& scope = MutableProcessScope(scopeId); - scope.UpdateLimits(processCpuLimits); +std::shared_ptr TProcessCategory::UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { + auto scope = GetProcessScopePtrVerified(scopeId); + scope->UpdateLimits(processCpuLimits); return scope; } -TProcessScope& TProcessCategory::UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { +std::shared_ptr TProcessCategory::UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { if (Scopes.contains(scopeId)) { return UpdateScope(scopeId, processCpuLimits); } else { @@ -74,8 +80,7 @@ TProcessScope& TProcessCategory::UpsertScope(const TString& scopeId, const TCPUL void TProcessCategory::UnregisterScope(const TString& name) { auto it = Scopes.find(name); AFL_VERIFY(it != Scopes.end()); - AFL_VERIFY(!it->second->GetProcessesCount()); Scopes.erase(it); } -} +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/category.h b/ydb/core/tx/conveyor_composite/service/category.h index 78c8f9b6b61e..1a78708b6eb0 100644 --- a/ydb/core/tx/conveyor_composite/service/category.h +++ b/ydb/core/tx/conveyor_composite/service/category.h @@ -1,6 +1,7 @@ #pragma once #include "common.h" #include "counters.h" +#include "process.h" #include "scope.h" #include "worker.h" @@ -12,38 +13,76 @@ class TProcessCategory: public TNonCopyable { private: const ESpecialTaskCategory Category; std::shared_ptr CPUUsage = std::make_shared(nullptr); - TPositiveControlInteger WaitingTasksCount; + std::shared_ptr WaitingTasksCount = std::make_shared(); YDB_READONLY_DEF(std::shared_ptr, Counters); THashMap> Scopes; + THashMap> Processes; + THashMap> ProcessesWithTasks; const NConfig::TCategory Config; public: + ui32 GetWaitingQueueSize() const { + return WaitingTasksCount->Val(); + } TProcessCategory(const NConfig::TCategory& config, TCounters& counters) : Category(config.GetCategory()) , Config(config) { Counters = counters.GetCategorySignals(Category); + RegisterProcess(0, RegisterScope("DEFAULT", TCPULimitsConfig(1000, 1000))); + Counters->WaitingQueueSizeLimit->Set(config.GetQueueSizeLimit()); } - ESpecialTaskCategory GetCategory() const { - return Category; + ~TProcessCategory() { + UnregisterProcess(0); + } + + void RegisterTask(const ui64 internalProcessId, std::shared_ptr&& task) { + auto it = Processes.find(internalProcessId); + AFL_VERIFY(it != Processes.end())("process_id", internalProcessId); + it->second->RegisterTask(std::move(task), Category); + if (it->second->GetTasksCount() == 1) { + AFL_VERIFY(ProcessesWithTasks.emplace(internalProcessId, it->second).second); + } } void PutTaskResult(TWorkerTaskResult&& result) { - const TString id = result.GetScopeId(); - if (TProcessScope* scope = MutableProcessScopeOptional(id)) { - scope->PutTaskResult(std::move(result)); + const ui64 internalProcessId = result.GetProcessId(); + auto it = Processes.find(internalProcessId); + if (it == Processes.end()) { + return; } + it->second->PutTaskResult(std::move(result)); + } + + void RegisterProcess(const ui64 internalProcessId, std::shared_ptr&& scope) { + scope->IncProcesses(); + AFL_VERIFY(Processes.emplace(internalProcessId, std::make_shared(internalProcessId, std::move(scope), WaitingTasksCount)).second); + } + + void UnregisterProcess(const ui64 processId) { + auto it = Processes.find(processId); + AFL_VERIFY(it != Processes.end()); + ProcessesWithTasks.erase(processId); + if (it->second->GetScope()->DecProcesses()) { + AFL_VERIFY(Scopes.erase(it->second->GetScope()->GetScopeId())); + } + Processes.erase(it); + } + + ESpecialTaskCategory GetCategory() const { + return Category; } bool HasTasks() const; void DoQuant(const TMonotonic newStart); - TWorkerTask ExtractTaskWithPrediction(); + TWorkerTask ExtractTaskWithPrediction(const std::shared_ptr& counters); TProcessScope& MutableProcessScope(const TString& scopeName); TProcessScope* MutableProcessScopeOptional(const TString& scopeName); - TProcessScope& RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); - TProcessScope& UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + std::shared_ptr GetProcessScopePtrVerified(const TString& scopeName) const; + std::shared_ptr RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + std::shared_ptr UpsertScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); - TProcessScope& UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); + std::shared_ptr UpdateScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits); void UnregisterScope(const TString& name); }; diff --git a/ydb/core/tx/conveyor_composite/service/common.cpp b/ydb/core/tx/conveyor_composite/service/common.cpp index 46a9129dc635..766662a80bef 100644 --- a/ydb/core/tx/conveyor_composite/service/common.cpp +++ b/ydb/core/tx/conveyor_composite/service/common.cpp @@ -30,8 +30,4 @@ void TCPUUsage::Cut(const TMonotonic start) { } } -TCPUGroup::~TCPUGroup() { - AFL_VERIFY(ProcessesCount == 0); -} - } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/common.h b/ydb/core/tx/conveyor_composite/service/common.h index b35e49a17f49..8faa4e1cc6b2 100644 --- a/ydb/core/tx/conveyor_composite/service/common.h +++ b/ydb/core/tx/conveyor_composite/service/common.h @@ -97,7 +97,6 @@ class TCPUUsage { class TCPUGroup { YDB_ACCESSOR_DEF(double, CPUThreadsLimit); YDB_ACCESSOR(double, Weight, 1); - TPositiveControlInteger ProcessesCount; public: using TPtr = std::shared_ptr; @@ -108,20 +107,6 @@ class TCPUGroup { { } - ~TCPUGroup(); - - ui32 GetProcessesCount() const { - return ProcessesCount.Val(); - } - - bool DecProcesses() { - --ProcessesCount; - return ProcessesCount == 0; - } - - void IncProcesses() { - ++ProcessesCount; - } }; } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/counters.cpp b/ydb/core/tx/conveyor_composite/service/counters.cpp index 59e2f7020887..386c64427507 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.cpp +++ b/ydb/core/tx/conveyor_composite/service/counters.cpp @@ -2,11 +2,14 @@ namespace NKikimr::NConveyorComposite { -TCounters::TCounters(const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals) - : TBase("CompositeConveyor/" + conveyorName, baseSignals) - , ProcessesCount(TBase::GetValue("Processes/Count")) - , WaitingQueueSize(TBase::GetValue("WaitingQueueSize")) - , WaitingQueueSizeLimit(TBase::GetValue("WaitingQueueSizeLimit")) +TWorkersPoolCounters::TWorkersPoolCounters(const TString& poolName, const NColumnShard::TCommonCountersOwner& owner) + : TBase(owner, "pool_name", poolName) + , PackSizeHistogram(TBase::GetHistogram("ExecutionPack/Count", NMonitoring::LinearHistogram(25, 1, 1))) + , PackExecuteHistogram(TBase::GetHistogram("PackExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendBackHistogram(TBase::GetHistogram("SendBack/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendFwdHistogram(TBase::GetHistogram("SendForward/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , SendBackDuration(TBase::GetDeriviative("SendBack/Duration/Us")) + , SendFwdDuration(TBase::GetDeriviative("SendForward/Duration/Us")) , AvailableWorkersCount(TBase::GetValue("AvailableWorkersCount")) , WorkersCountLimit(TBase::GetValue("WorkersCountLimit")) , AmountCPULimit(TBase::GetValue("AmountCPULimit")) @@ -16,16 +19,15 @@ TCounters::TCounters(const TString& conveyorName, TIntrusivePtr<::NMonitoring::T , WaitWorkerRate(TBase::GetDeriviative("WaitWorker")) , UseWorkerRate(TBase::GetDeriviative("UseWorker")) , ChangeCPULimitRate(TBase::GetDeriviative("ChangeCPULimit")) + , NoTasks(TBase::GetDeriviative("NoTasks")) { +} + +TWPCategorySignals::TWPCategorySignals(NColumnShard::TCommonCountersOwner& base, const ESpecialTaskCategory cat) + : TBase(base, "wp_category", ::ToString(cat)) + , Category(cat) , WaitingHistogram(TBase::GetHistogram("Waiting/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) - , PackHistogram(TBase::GetHistogram("ExecutionPack/Count", NMonitoring::LinearHistogram(25, 1, 1))) - , PackExecuteHistogram(TBase::GetHistogram("PackExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) , TaskExecuteHistogram(TBase::GetHistogram("TaskExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) - , SendBackHistogram(TBase::GetHistogram("SendBack/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) - , SendFwdHistogram(TBase::GetHistogram("SendForward/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) - , ReceiveTaskHistogram(TBase::GetHistogram("ReceiveTask/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) - , SendBackDuration(TBase::GetDeriviative("SendBack/Duration/Us")) - , SendFwdDuration(TBase::GetDeriviative("SendForward/Duration/Us")) , ExecuteDuration(TBase::GetDeriviative("Execute/Duration/Us")) { } -} +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/counters.h b/ydb/core/tx/conveyor_composite/service/counters.h index e6b7f38fc8f2..cc85191907db 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.h +++ b/ydb/core/tx/conveyor_composite/service/counters.h @@ -11,14 +11,36 @@ using TTaskSignals = NConveyor::TTaskSignals; class TCategorySignals: public NColumnShard::TCommonCountersOwner { private: using TBase = NColumnShard::TCommonCountersOwner; - THashMap> TaskClassSignals; YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); public: + const ::NMonitoring::TDynamicCounters::TCounterPtr ProcessesCount; + + const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSize; + const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSizeLimit; + TCategorySignals(NColumnShard::TCommonCountersOwner& base, const ESpecialTaskCategory cat) : TBase(base, "category", ::ToString(cat)) - , Category(cat) { + , Category(cat) + , ProcessesCount(TBase::GetValue("ProcessesCount")) + , WaitingQueueSize(TBase::GetValue("WaitingQueueSize")) + , WaitingQueueSizeLimit(TBase::GetValue("WaitingQueueSizeLimit")) { } +}; + +class TWPCategorySignals: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + THashMap> TaskClassSignals; + YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); + +public: + const ::NMonitoring::THistogramPtr WaitingHistogram; + const ::NMonitoring::THistogramPtr TaskExecuteHistogram; + + const ::NMonitoring::TDynamicCounters::TCounterPtr ExecuteDuration; + + TWPCategorySignals(NColumnShard::TCommonCountersOwner& base, const ESpecialTaskCategory cat); std::shared_ptr GetTaskSignals(const TString& taskClassName) { auto it = TaskClassSignals.find(taskClassName); @@ -29,18 +51,19 @@ class TCategorySignals: public NColumnShard::TCommonCountersOwner { } }; -class TCounters: public NColumnShard::TCommonCountersOwner { +class TWorkersPoolCounters: public NColumnShard::TCommonCountersOwner { private: using TBase = NColumnShard::TCommonCountersOwner; - THashMap> CategorySignals; + THashMap> CategorySignals; public: - const ::NMonitoring::TDynamicCounters::TCounterPtr ProcessesCount; - - const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSize; - const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueSizeLimit; + const ::NMonitoring::THistogramPtr PackSizeHistogram; + const ::NMonitoring::THistogramPtr PackExecuteHistogram; + const ::NMonitoring::THistogramPtr SendBackHistogram; + const ::NMonitoring::THistogramPtr SendFwdHistogram; - const ::NMonitoring::TDynamicCounters::TCounterPtr InProgressSize; + const ::NMonitoring::TDynamicCounters::TCounterPtr SendBackDuration; + const ::NMonitoring::TDynamicCounters::TCounterPtr SendFwdDuration; const ::NMonitoring::TDynamicCounters::TCounterPtr AvailableWorkersCount; const ::NMonitoring::TDynamicCounters::TCounterPtr WorkersCountLimit; @@ -52,18 +75,35 @@ class TCounters: public NColumnShard::TCommonCountersOwner { const ::NMonitoring::TDynamicCounters::TCounterPtr WaitWorkerRate; const ::NMonitoring::TDynamicCounters::TCounterPtr UseWorkerRate; const ::NMonitoring::TDynamicCounters::TCounterPtr ChangeCPULimitRate; + const ::NMonitoring::TDynamicCounters::TCounterPtr NoTasks; - const ::NMonitoring::THistogramPtr WaitingHistogram; - const ::NMonitoring::THistogramPtr PackHistogram; - const ::NMonitoring::THistogramPtr PackExecuteHistogram; - const ::NMonitoring::THistogramPtr TaskExecuteHistogram; - const ::NMonitoring::THistogramPtr SendBackHistogram; - const ::NMonitoring::THistogramPtr SendFwdHistogram; + TWorkersPoolCounters(const TString& poolName, const NColumnShard::TCommonCountersOwner& owner); + + std::shared_ptr GetCategorySignals(const ESpecialTaskCategory cat) { + auto it = CategorySignals.find(cat); + if (it == CategorySignals.end()) { + it = CategorySignals.emplace(cat, std::make_shared(*this, cat)).first; + } + return it->second; + } +}; + +class TCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + THashMap> CategorySignals; + THashMap> WorkersPoolSignals; + +public: const ::NMonitoring::THistogramPtr ReceiveTaskHistogram; + const ::NMonitoring::TDynamicCounters::TCounterPtr ReceiveTaskDuration; - const ::NMonitoring::TDynamicCounters::TCounterPtr SendBackDuration; - const ::NMonitoring::TDynamicCounters::TCounterPtr SendFwdDuration; - const ::NMonitoring::TDynamicCounters::TCounterPtr ExecuteDuration; + TCounters(const TString& module, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals) + : TBase(module, baseSignals) + , ReceiveTaskHistogram(TBase::GetHistogram("ReceiveTask/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , ReceiveTaskDuration(TBase::GetDeriviative("ReceiveTask/Duration/Us")) + { + } std::shared_ptr GetCategorySignals(const ESpecialTaskCategory cat) { auto it = CategorySignals.find(cat); @@ -73,7 +113,13 @@ class TCounters: public NColumnShard::TCommonCountersOwner { return it->second; } - TCounters(const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseSignals); + std::shared_ptr GetWorkersPoolSignals(const TString& poolName) { + auto it = WorkersPoolSignals.find(poolName); + if (it == WorkersPoolSignals.end()) { + it = WorkersPoolSignals.emplace(poolName, std::make_shared(poolName, *this)).first; + } + return it->second; + } }; } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/events.h b/ydb/core/tx/conveyor_composite/service/events.h index dcafb80c6755..6f9b0697a323 100644 --- a/ydb/core/tx/conveyor_composite/service/events.h +++ b/ydb/core/tx/conveyor_composite/service/events.h @@ -1,5 +1,6 @@ #pragma once #include "counters.h" +#include "scope.h" #include @@ -15,14 +16,15 @@ class TWorkerTaskContext { YDB_READONLY(TMonotonic, CreateInstant, TMonotonic::Now()); YDB_READONLY_DEF(TDuration, PredictedDuration); YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); - YDB_READONLY_DEF(TString, ScopeId); + YDB_READONLY_DEF(std::shared_ptr, Scope); YDB_READONLY(ui64, ProcessId, 0); public: - TWorkerTaskContext(const TDuration prediction, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + TWorkerTaskContext( + const TDuration prediction, const ESpecialTaskCategory category, const std::shared_ptr& scope, const ui64 processId) : PredictedDuration(prediction) , Category(category) - , ScopeId(scopeId) + , Scope(scope) , ProcessId(processId) { } }; @@ -42,7 +44,6 @@ class TWorkerTaskResult: public TWorkerTaskContext { TDuration GetDuration() const { return Finish - Start; } - }; class TWorkerTask: public TWorkerTaskContext { @@ -56,15 +57,41 @@ class TWorkerTask: public TWorkerTaskContext { return TWorkerTaskResult(*this, start, finish); } - TWorkerTask(const ITask::TPtr& task, const TDuration prediction, const ESpecialTaskCategory category, const TString& scopeId, - const std::shared_ptr& taskSignals, const ui64 processId) - : TBase(prediction, category, scopeId, processId) + TWorkerTask(const ITask::TPtr& task, const TDuration prediction, const ESpecialTaskCategory category, + const std::shared_ptr& scope, const std::shared_ptr& taskSignals, const ui64 processId) + : TBase(prediction, category, scope, processId) , Task(task) , TaskSignals(taskSignals) { Y_ABORT_UNLESS(task); } - bool operator<(const TWorkerTask& wTask) const { + TWorkerTask(TWorkerTaskContext&& context, ITask::TPtr&& task, std::shared_ptr&& taskSignals) + : TBase(std::move(context)) + , Task(std::move(task)) + , TaskSignals(std::move(taskSignals)) { + AFL_VERIFY(Task); + AFL_VERIFY(TaskSignals); + } +}; + +class TWorkerTaskPrepare: public TWorkerTaskContext { +private: + using TBase = TWorkerTaskContext; + YDB_READONLY_DEF(ITask::TPtr, Task); + +public: + TWorkerTask BuildTask(std::shared_ptr&& signals) && { + return TWorkerTask(std::move(*this), std::move(Task), std::move(signals)); + } + + TWorkerTaskPrepare(const ITask::TPtr& task, const TDuration prediction, const ESpecialTaskCategory category, + const std::shared_ptr& scope, const ui64 processId) + : TBase(prediction, category, scope, processId) + , Task(task) { + AFL_VERIFY(task); + } + + bool operator<(const TWorkerTaskPrepare& wTask) const { return Task->GetPriority() < wTask.Task->GetPriority(); } }; @@ -105,6 +132,10 @@ struct TEvInternal { YDB_READONLY(ui64, WorkersPoolId, 0); public: + const std::vector& GetResults() const { + return Results; + } + std::vector&& DetachResults() { return std::move(Results); } diff --git a/ydb/core/tx/conveyor_composite/service/manager.h b/ydb/core/tx/conveyor_composite/service/manager.h index 30c0cc7666f6..f1184b265e73 100644 --- a/ydb/core/tx/conveyor_composite/service/manager.h +++ b/ydb/core/tx/conveyor_composite/service/manager.h @@ -7,7 +7,6 @@ namespace NKikimr::NConveyorComposite { class TTasksManager { private: - const TString ConveyorName; std::vector> WorkerPools; std::vector> Categories; const NActors::TActorId DistributorActorId; @@ -18,7 +17,6 @@ class TTasksManager { TString DebugString() const { TStringBuilder sb; sb << "{"; - sb << ConveyorName << ":"; for (auto&& wp : WorkerPools) { sb << wp->GetMaxWorkerThreads() << ","; } @@ -33,9 +31,8 @@ class TTasksManager { } } - TTasksManager(const TString& convName, const NConfig::TConfig& config, const NActors::TActorId distributorActorId, TCounters& counters) - : ConveyorName(convName) - , DistributorActorId(distributorActorId) + TTasksManager(const TString& /*convName*/, const NConfig::TConfig& config, const NActors::TActorId distributorActorId, TCounters& counters) + : DistributorActorId(distributorActorId) , Config(config) , Counters(counters) { @@ -43,7 +40,8 @@ class TTasksManager { Categories.emplace_back(std::make_shared(Config.GetCategoryConfig(i), Counters)); } for (auto&& i : Config.GetWorkerPools()) { - WorkerPools.emplace_back(std::make_shared(ConveyorName, distributorActorId, i, Counters, Categories)); + WorkerPools.emplace_back(std::make_shared( + i.GetName(), distributorActorId, i, Counters.GetWorkersPoolSignals(i.GetName()), Categories)); } } diff --git a/ydb/core/tx/conveyor_composite/service/process.h b/ydb/core/tx/conveyor_composite/service/process.h index cbc58bb4de47..d8ebe6a2d2dc 100644 --- a/ydb/core/tx/conveyor_composite/service/process.h +++ b/ydb/core/tx/conveyor_composite/service/process.h @@ -1,5 +1,6 @@ #pragma once #include "common.h" +#include "scope.h" #include "worker.h" #include @@ -8,6 +9,7 @@ #include #include #include +#include #include #include @@ -18,15 +20,16 @@ namespace NKikimr::NConveyorComposite { class TDequePriorityFIFO { private: - std::map> Tasks; + std::map> Tasks; ui32 Size = 0; public: - void push(const TWorkerTask& task) { - Tasks[(ui32)task.GetTask()->GetPriority()].emplace_back(task); + void push(TWorkerTaskPrepare&& task) { + auto priority = (ui32)task.GetTask()->GetPriority(); + Tasks[priority].emplace_back(std::move(task)); ++Size; } - TWorkerTask pop() { + TWorkerTaskPrepare pop() { Y_ABORT_UNLESS(Size); auto result = std::move(Tasks.rbegin()->second.front()); Tasks.rbegin()->second.pop_front(); @@ -63,15 +66,22 @@ class TProcessOrdered { } }; -class TProcess: public TMoveOnly { +class TProcess: public TNonCopyable, public NColumnShard::TMonitoringObjectsCounter { private: YDB_READONLY(ui64, ProcessId, 0); YDB_READONLY_DEF(std::shared_ptr, CPUUsage); YDB_ACCESSOR_DEF(TDequePriorityFIFO, Tasks); + YDB_READONLY_DEF(std::shared_ptr, Scope); + + std::shared_ptr WaitingTasksCount; TAverageCalcer AverageTaskDuration; ui32 LinksCount = 0; public: + ~TProcess() { + WaitingTasksCount->Sub(Tasks.size()); + } + void DoQuant(const TMonotonic newStart) { CPUUsage->Cut(newStart); } @@ -80,10 +90,15 @@ class TProcess: public TMoveOnly { return Tasks.size(); } - TWorkerTask ExtractTaskWithPrediction() { + ui32 GetTasksCount() const { + return Tasks.size(); + } + + TWorkerTask ExtractTaskWithPrediction(const std::shared_ptr& signals) { auto result = Tasks.pop(); CPUUsage->AddPredicted(result.GetPredictedDuration()); - return result; + WaitingTasksCount->Dec(); + return std::move(result).BuildTask(signals->GetTaskSignals(result.GetTask()->GetTaskClassIdentifier())); } void PutTaskResult(TWorkerTaskResult&& result) { @@ -105,17 +120,20 @@ class TProcess: public TMoveOnly { ++LinksCount; } - TProcess(const ui64 processId, const std::shared_ptr& scopeUsage) + TProcess( + const ui64 processId, const std::shared_ptr& scope, const std::shared_ptr& waitingTasksCount) : ProcessId(processId) - { - CPUUsage = std::make_shared(scopeUsage); + , Scope(scope) + , WaitingTasksCount(waitingTasksCount) { + AFL_VERIFY(WaitingTasksCount); + CPUUsage = std::make_shared(Scope->GetCPUUsage()); IncRegistration(); } - void RegisterTask(const std::shared_ptr& task, const TString& scopeId, const std::shared_ptr& signals) { - TWorkerTask wTask(task, AverageTaskDuration.GetValue(), signals->GetCategory(), scopeId, - signals->GetTaskSignals(task->GetTaskClassIdentifier()), ProcessId); + void RegisterTask(const std::shared_ptr& task, const ESpecialTaskCategory category) { + TWorkerTaskPrepare wTask(task, AverageTaskDuration.GetValue(), category, Scope, ProcessId); Tasks.push(std::move(wTask)); + WaitingTasksCount->Inc(); } }; diff --git a/ydb/core/tx/conveyor_composite/service/scope.cpp b/ydb/core/tx/conveyor_composite/service/scope.cpp index 0613ee69e20d..255545c3d482 100644 --- a/ydb/core/tx/conveyor_composite/service/scope.cpp +++ b/ydb/core/tx/conveyor_composite/service/scope.cpp @@ -2,37 +2,4 @@ namespace NKikimr::NConveyorComposite { -bool TProcessScope::HasTasks() const { - for (auto&& i : Processes) { - if (i.second.HasTasks()) { - return true; - } - } - return false; -} - -TWorkerTask TProcessScope::ExtractTaskWithPrediction() { - TProcess* pMin = nullptr; - TDuration dMin = TDuration::Max(); - for (auto&& [_, p] : Processes) { - if (!p.HasTasks()) { - continue; - } - const TDuration d = p.GetCPUUsage()->CalcWeight(p.GetWeight()); - if (!pMin || d < dMin) { - pMin = &p; - dMin = d; - } - } - AFL_VERIFY(pMin)("size", Processes.size()); - return pMin->ExtractTaskWithPrediction(); -} - -void TProcessScope::DoQuant(const TMonotonic newStart) { - CPUUsage->Cut(newStart); - for (auto&& i : Processes) { - i.second.DoQuant(newStart); - } -} - } diff --git a/ydb/core/tx/conveyor_composite/service/scope.h b/ydb/core/tx/conveyor_composite/service/scope.h index b95a15dad202..33016b3a2f6b 100644 --- a/ydb/core/tx/conveyor_composite/service/scope.h +++ b/ydb/core/tx/conveyor_composite/service/scope.h @@ -1,70 +1,59 @@ #pragma once #include "common.h" -#include "process.h" + +#include + +#include namespace NKikimr::NConveyorComposite { -class TProcessScope: public TNonCopyable { +class TProcessScope: public TNonCopyable, public NColumnShard::TMonitoringObjectsCounter { private: + YDB_READONLY_DEF(TString, ScopeId); YDB_READONLY_DEF(std::shared_ptr, CPUUsage); TCPUGroup::TPtr ScopeLimits; - THashMap Processes; + TPositiveControlInteger CountInFlight; + TPositiveControlInteger CountProcesses; public: - double GetWeight() const { - return ScopeLimits->GetWeight(); + void IncProcesses() { + CountProcesses.Inc(); } - ui32 GetProcessesCount() const { - return Processes.size(); + bool DecProcesses() { + return CountProcesses.Dec() == 0; } - bool HasTasks() const; - TWorkerTask ExtractTaskWithPrediction(); - - void DoQuant(const TMonotonic newStart); - - void UpdateLimits(const TCPULimitsConfig& processCpuLimits) { - ScopeLimits->SetCPUThreadsLimit(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); - ScopeLimits->SetWeight(processCpuLimits.GetWeight()); + ui32 GetCountInFlight() const { + return CountInFlight.Val(); } - void PutTaskResult(TWorkerTaskResult&& result) { - const ui64 id = result.GetProcessId(); - if (auto* process = MutableProcessOptional(id)) { - process->PutTaskResult(std::move(result)); - } + bool CheckToRun() const { + return CountInFlight.Val() + 1 <= std::ceil(ScopeLimits->GetCPUThreadsLimit()); } - TProcessScope(TCPUGroup::TPtr&& limits, const std::shared_ptr& categoryScope) - : CPUUsage(std::make_shared(categoryScope)) - , ScopeLimits(std::move(limits)) { + void IncInFlight() { + AFL_VERIFY(CountInFlight.Inc() <= std::ceil(ScopeLimits->GetCPUThreadsLimit()))("in_flight", CountInFlight.Inc())( + "limit", ScopeLimits->GetCPUThreadsLimit()); } - TProcess& MutableProcessVerified(const ui64 processId) { - auto it = Processes.find(processId); - AFL_VERIFY(it != Processes.end()); - return it->second; + void DecInFlight() { + CountInFlight.Dec(); } - TProcess* MutableProcessOptional(const ui64 processId) { - auto it = Processes.find(processId); - if (it != Processes.end()) { - return &it->second; - } else { - return nullptr; - } + double GetWeight() const { + return ScopeLimits->GetWeight(); } - void RegisterProcess(const ui64 processId) { - TProcess process(processId, CPUUsage); - AFL_VERIFY(Processes.emplace(processId, std::move(process)).second); - ScopeLimits->IncProcesses(); + void UpdateLimits(const TCPULimitsConfig& processCpuLimits) { + ScopeLimits->SetCPUThreadsLimit(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); + ScopeLimits->SetWeight(processCpuLimits.GetWeight()); } - bool UnregisterProcess(const ui64 processId) { - AFL_VERIFY(Processes.erase(processId)); - return ScopeLimits->DecProcesses(); + TProcessScope(const TString& scopeId, TCPUGroup::TPtr&& limits, const std::shared_ptr& categoryScope) + : ScopeId(scopeId) + , CPUUsage(std::make_shared(categoryScope)) + , ScopeLimits(std::move(limits)) { } }; diff --git a/ydb/core/tx/conveyor_composite/service/service.cpp b/ydb/core/tx/conveyor_composite/service/service.cpp index 1195f7c1a081..8d68579810a0 100644 --- a/ydb/core/tx/conveyor_composite/service/service.cpp +++ b/ydb/core/tx/conveyor_composite/service/service.cpp @@ -6,10 +6,9 @@ namespace NKikimr::NConveyorComposite { -TDistributor::TDistributor( - const NConfig::TConfig& config, const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) +TDistributor::TDistributor(const NConfig::TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) : Config(config) - , ConveyorName(conveyorName) + , ConveyorName("COMPOSITE_CONVEYOR") , Counters(ConveyorName, conveyorSignals) { } @@ -32,38 +31,45 @@ void TDistributor::HandleMain(NActors::TEvents::TEvWakeup::TPtr& evExt) { } void TDistributor::HandleMain(TEvInternal::TEvTaskProcessedResult::TPtr& evExt) { - TWorkersPool& workersPool = Manager->MutableWorkersPool(evExt->Get()->GetWorkersPoolId()); - workersPool.AddDeliveryDuration(evExt->Get()->GetForwardSendDuration() + (TMonotonic::Now() - evExt->Get()->GetConstructInstant())); - workersPool.ReleaseWorker(evExt->Get()->GetWorkerIdx()); - for (auto&& i : evExt->Get()->DetachResults()) { - workersPool.PutTaskResult(std::move(i)); - } + auto& ev = *evExt->Get(); + const TDuration backSendDuration = (TMonotonic::Now() - ev.GetConstructInstant()); + TWorkersPool& workersPool = Manager->MutableWorkersPool(ev.GetWorkersPoolId()); + workersPool.GetCounters()->PackExecuteHistogram->Collect( + (ev.GetResults().back().GetFinish() - ev.GetResults().front().GetStart()).MicroSeconds()); + workersPool.GetCounters()->PackSizeHistogram->Collect(ev.GetResults().size()); + workersPool.GetCounters()->SendBackHistogram->Collect(backSendDuration.MicroSeconds()); + workersPool.GetCounters()->SendFwdHistogram->Collect(ev.GetForwardSendDuration().MicroSeconds()); + + workersPool.GetCounters()->SendBackDuration->Add(backSendDuration.MicroSeconds()); + workersPool.GetCounters()->SendFwdDuration->Add(ev.GetForwardSendDuration().MicroSeconds()); + + workersPool.AddDeliveryDuration(ev.GetForwardSendDuration() + backSendDuration); + workersPool.ReleaseWorker(ev.GetWorkerIdx()); + workersPool.PutTaskResults(ev.DetachResults()); if (workersPool.HasTasks()) { AFL_VERIFY(workersPool.DrainTasks()); } } void TDistributor::HandleMain(TEvExecution::TEvRegisterProcess::TPtr& ev) { - Manager->MutableCategoryVerified(ev->Get()->GetCategory()) - .UpsertScope(ev->Get()->GetScopeId(), ev->Get()->GetCPULimits()) - .RegisterProcess(ev->Get()->GetProcessId()); + auto& cat = Manager->MutableCategoryVerified(ev->Get()->GetCategory()); + std::shared_ptr scope = cat.UpsertScope(ev->Get()->GetScopeId(), ev->Get()->GetCPULimits()); + cat.RegisterProcess(ev->Get()->GetInternalProcessId(), std::move(scope)); } void TDistributor::HandleMain(TEvExecution::TEvUnregisterProcess::TPtr& ev) { auto* evData = ev->Get(); - if (Manager->MutableCategoryVerified(evData->GetCategory()) - .MutableProcessScope(evData->GetScopeId()) - .UnregisterProcess(evData->GetProcessId())) { - Manager->MutableCategoryVerified(evData->GetCategory()).UnregisterScope(evData->GetScopeId()); - } + Manager->MutableCategoryVerified(evData->GetCategory()).UnregisterProcess(evData->GetInternalProcessId()); } void TDistributor::HandleMain(TEvExecution::TEvNewTask::TPtr& ev) { + const TDuration d = TMonotonic::Now() - ev->Get()->GetConstructInstant(); + Counters.ReceiveTaskDuration->Add(d.MicroSeconds()); + Counters.ReceiveTaskHistogram->Collect(d.MicroSeconds()); auto& cat = Manager->MutableCategoryVerified(ev->Get()->GetCategory()); - cat.MutableProcessScope(ev->Get()->GetScopeId()) - .MutableProcessVerified(ev->Get()->GetProcessId()) - .RegisterTask(ev->Get()->GetTask(), ev->Get()->GetScopeId(), cat.GetCounters()); + cat.RegisterTask(ev->Get()->GetInternalProcessId(), ev->Get()->DetachTask()); Y_UNUSED(Manager->DrainTasks()); + cat.GetCounters()->WaitingQueueSize->Set(cat.GetWaitingQueueSize()); } } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/service.h b/ydb/core/tx/conveyor_composite/service/service.h index 159fff0e4168..4794c9a91bae 100644 --- a/ydb/core/tx/conveyor_composite/service/service.h +++ b/ydb/core/tx/conveyor_composite/service/service.h @@ -57,7 +57,7 @@ class TDistributor: public TActorBootstrapped { } } - TDistributor(const NConfig::TConfig& config, const TString& conveyorName, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals); + TDistributor(const NConfig::TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals); ~TDistributor(); diff --git a/ydb/core/tx/conveyor_composite/service/worker.cpp b/ydb/core/tx/conveyor_composite/service/worker.cpp index 3ab805fe3ee7..1b126deb3f08 100644 --- a/ydb/core/tx/conveyor_composite/service/worker.cpp +++ b/ydb/core/tx/conveyor_composite/service/worker.cpp @@ -56,8 +56,6 @@ void TWorker::HandleMain(TEvInternal::TEvNewTask::TPtr& ev) { AFL_VERIFY(!WaitWakeUp); const TMonotonic now = TMonotonic::Now(); ForwardDuration = now - ev->Get()->GetConstructInstant(); - SendFwdHistogram->Collect(ForwardDuration->MicroSeconds()); - SendFwdDuration->Add(ForwardDuration->MicroSeconds()); ExecuteTask(ev->Get()->ExtractTasks()); } diff --git a/ydb/core/tx/conveyor_composite/service/worker.h b/ydb/core/tx/conveyor_composite/service/worker.h index c8209c1e7975..e310f420fa26 100644 --- a/ydb/core/tx/conveyor_composite/service/worker.h +++ b/ydb/core/tx/conveyor_composite/service/worker.h @@ -27,8 +27,6 @@ class TWorker: public NActors::TActorBootstrapped { const ui64 WorkersPoolId; std::optional ExecutionDuration; std::vector Results; - const ::NMonitoring::THistogramPtr SendFwdHistogram; - const ::NMonitoring::TDynamicCounters::TCounterPtr SendFwdDuration; TDuration GetWakeupDuration() const; void ExecuteTask(std::vector&& workerTasks); void HandleMain(TEvInternal::TEvNewTask::TPtr& ev); @@ -50,17 +48,14 @@ class TWorker: public NActors::TActorBootstrapped { Become(&TWorker::StateMain); } - TWorker(const TString& conveyorName, const double cpuHardLimit, const NActors::TActorId& distributorId, const ui64 workerIdx, - const ui64 workersPoolId, const ::NMonitoring::THistogramPtr sendFwdHistogram, - const ::NMonitoring::TDynamicCounters::TCounterPtr sendFwdDuration) - : TBase("CONVEYOR::" + conveyorName + "::WORKER") + TWorker(const TString& poolName, const double cpuHardLimit, const NActors::TActorId& distributorId, const ui64 workerIdx, + const ui64 workersPoolId) + : TBase("COMPOSITE_CONVEYOR::" + poolName + "::WORKER") , CPUHardLimit(cpuHardLimit) , CPUSoftLimit(cpuHardLimit) , DistributorId(distributorId) , WorkerIdx(workerIdx) - , WorkersPoolId(workersPoolId) - , SendFwdHistogram(sendFwdHistogram) - , SendFwdDuration(sendFwdDuration) { + , WorkersPoolId(workersPoolId) { AFL_VERIFY(0 < CPUHardLimit); AFL_VERIFY(CPUHardLimit <= 1); } diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp index 8a7221869536..099eec259ecd 100644 --- a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp @@ -1,27 +1,28 @@ #include "workers_pool.h" +#include + namespace NKikimr::NConveyorComposite { -TWorkersPool::TWorkersPool(const TString& conveyorName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, - const TCounters& counters, const std::vector>& categories) - : WorkersCount(config.GetWorkersCount()) +TWorkersPool::TWorkersPool(const TString& poolName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, + const std::shared_ptr& counters, const std::vector>& categories) + : WorkersCount(config.GetWorkersCountInfo().GetThreadsCount(NKqp::TStagePredictor::GetUsableThreads())) , Counters(counters) { Workers.reserve(WorkersCount); for (auto&& i : config.GetLinks()) { AFL_VERIFY((ui64)i.GetCategory() < categories.size()); - Processes.emplace_back(TWeightedCategory(i.GetWeight(), categories[(ui64)i.GetCategory()])); + Processes.emplace_back(TWeightedCategory(i.GetWeight(), categories[(ui64)i.GetCategory()], Counters->GetCategorySignals(i.GetCategory()))); } AFL_VERIFY(Processes.size()); for (ui32 i = 0; i < WorkersCount; ++i) { - Workers.emplace_back(std::make_unique(conveyorName, config.GetWorkerCPUUsage(i), distributorId, i, - config.GetWorkersPoolId(), Counters.SendFwdHistogram, Counters.SendFwdDuration)); + Workers.emplace_back(std::make_unique( + poolName, config.GetWorkerCPUUsage(i, NKqp::TStagePredictor::GetUsableThreads()), distributorId, i, config.GetWorkersPoolId())); ActiveWorkersIdx.emplace_back(i); } - AFL_VERIFY(WorkersCount)("name", conveyorName)("action", "conveyor_registered")("config", config.DebugString())("actor_id", distributorId)( + AFL_VERIFY(WorkersCount)("name", poolName)("action", "conveyor_registered")("config", config.DebugString())("actor_id", distributorId)( "count", WorkersCount); - Counters.WaitingQueueSizeLimit->Set(config.GetWorkersCountDouble()); - Counters.AmountCPULimit->Set(0); - Counters.AvailableWorkersCount->Set(0); - Counters.WorkersCountLimit->Set(WorkersCount); + Counters->AmountCPULimit->Set(0); + Counters->AvailableWorkersCount->Set(0); + Counters->WorkersCountLimit->Set(WorkersCount); } bool TWorkersPool::HasFreeWorker() const { @@ -32,7 +33,7 @@ void TWorkersPool::RunTask(std::vector&& tasksBatch) { AFL_VERIFY(HasFreeWorker()); const auto workerIdx = ActiveWorkersIdx.back(); ActiveWorkersIdx.pop_back(); - Counters.AvailableWorkersCount->Set(ActiveWorkersIdx.size()); + Counters->AvailableWorkersCount->Set(ActiveWorkersIdx.size()); auto& worker = Workers[workerIdx]; worker.OnStartTask(); @@ -43,7 +44,7 @@ void TWorkersPool::ReleaseWorker(const ui32 workerIdx) { AFL_VERIFY(workerIdx < Workers.size()); Workers[workerIdx].OnStopTask(); ActiveWorkersIdx.emplace_back(workerIdx); - Counters.AvailableWorkersCount->Set(ActiveWorkersIdx.size()); + Counters->AvailableWorkersCount->Set(ActiveWorkersIdx.size()); } bool TWorkersPool::DrainTasks() { @@ -68,18 +69,49 @@ bool TWorkersPool::DrainTasks() { while (ActiveWorkersIdx.size() && Processes.front().GetCategory()->HasTasks()) { TDuration predicted = TDuration::Zero(); std::vector tasks; + THashMap> scopes; while ((tasks.empty() || predicted < DeliveringDuration.GetValue() * 10) && Processes.front().GetCategory()->HasTasks()) { std::pop_heap(Processes.begin(), Processes.end(), predHeap); - tasks.emplace_back(Processes.back().GetCategory()->ExtractTaskWithPrediction()); + tasks.emplace_back(Processes.back().GetCategory()->ExtractTaskWithPrediction(Processes.back().GetCounters())); Processes.back().GetCPUUsage()->AddPredicted(tasks.back().GetPredictedDuration()); predicted += tasks.back().GetPredictedDuration(); std::push_heap(Processes.begin(), Processes.end(), predHeap); + scopes.emplace(tasks.back().GetScope()->GetScopeId(), tasks.back().GetScope()); + } + for (auto&& i : scopes) { + i.second->IncInFlight(); } newTask = true; AFL_VERIFY(tasks.size()); RunTask(std::move(tasks)); } + if (!Processes.front().GetCategory()->HasTasks()) { + Counters->NoTasks->Add(1); + } return newTask; } +void TWorkersPool::PutTaskResults(std::vector&& result) { + // const ui32 catIdx = (ui32)result.GetCategory(); + std::set scopeIds; + for (auto&& t : result) { + bool found = false; + for (auto&& i : Processes) { + if (i.GetCategory()->GetCategory() == t.GetCategory()) { + i.GetCounters()->WaitingHistogram->Collect((t.GetStart() - t.GetCreateInstant()).MicroSeconds()); + i.GetCounters()->TaskExecuteHistogram->Collect((t.GetFinish() - t.GetStart()).MicroSeconds()); + i.GetCounters()->ExecuteDuration->Add((t.GetFinish() - t.GetStart()).MicroSeconds()); + found = true; + i.GetCPUUsage()->Exchange(t.GetPredictedDuration(), t.GetStart(), t.GetFinish()); + if (scopeIds.emplace(t.GetScope()->GetScopeId()).second) { + t.GetScope()->DecInFlight(); + } + i.GetCategory()->PutTaskResult(std::move(t)); + break; + } + } + AFL_VERIFY(found); + } +} + } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.h b/ydb/core/tx/conveyor_composite/service/workers_pool.h index 6fa9def6c9bd..3fb08814e0e9 100644 --- a/ydb/core/tx/conveyor_composite/service/workers_pool.h +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.h @@ -11,13 +11,16 @@ class TWeightedCategory { private: YDB_READONLY(std::shared_ptr, CPUUsage, std::make_shared(nullptr)); YDB_READONLY_DEF(std::shared_ptr, Category); + YDB_READONLY_DEF(std::shared_ptr, Counters); YDB_READONLY(double, Weight, 1); public: - TWeightedCategory(const double weight, const std::shared_ptr& cat) + TWeightedCategory(const double weight, const std::shared_ptr& cat, const std::shared_ptr& counters) : Category(cat) + , Counters(counters) , Weight(weight) { + AFL_VERIFY(Counters); AFL_VERIFY(cat); AFL_VERIFY(Weight); } @@ -53,7 +56,7 @@ class TWorkersPool { std::vector Processes; std::vector Workers; std::vector ActiveWorkersIdx; - TCounters Counters; + std::shared_ptr Counters; TAverageCalcer DeliveringDuration; std::deque DeliveryDurations; @@ -61,8 +64,12 @@ class TWorkersPool { static constexpr double Eps = 1e-6; using TPtr = std::shared_ptr; - TWorkersPool(const TString& conveyorName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, - const TCounters& counters, const std::vector>& categories); + TWorkersPool(const TString& poolName, const NActors::TActorId& distributorId, const NConfig::TWorkersPool& config, + const std::shared_ptr& counters, const std::vector>& categories); + + const std::shared_ptr& GetCounters() const { + return Counters; + } bool HasTasks() const { for (auto&& i : Processes) { @@ -79,19 +86,7 @@ class TWorkersPool { DeliveringDuration.Add(d); } - void PutTaskResult(TWorkerTaskResult&& result) { -// const ui32 catIdx = (ui32)result.GetCategory(); - for (auto&& i : Processes) { - if (i.GetCategory()->GetCategory() == result.GetCategory()) { -// AFL_VERIFY(catIdx < Processes.size()); -// AFL_VERIFY(Processes[catIdx]); - i.GetCPUUsage()->Exchange(result.GetPredictedDuration(), result.GetStart(), result.GetFinish()); - i.GetCategory()->PutTaskResult(std::move(result)); - return; - } - } - AFL_VERIFY(false); - } + void PutTaskResults(std::vector&& result); bool HasFreeWorker() const; void RunTask(std::vector&& tasksBatch); void ReleaseWorker(const ui32 workerIdx); diff --git a/ydb/core/tx/conveyor_composite/usage/common.cpp b/ydb/core/tx/conveyor_composite/usage/common.cpp index aeaaaa4af88e..b1090664ed32 100644 --- a/ydb/core/tx/conveyor_composite/usage/common.cpp +++ b/ydb/core/tx/conveyor_composite/usage/common.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "config.h" #include "events.h" #include @@ -11,7 +12,19 @@ void TProcessGuard::Finish() { Finished = true; if (ServiceActorId && NActors::TlsActivationContext) { auto& context = NActors::TActorContext::AsActorContext(); - context.Send(*ServiceActorId, new TEvExecution::TEvUnregisterProcess(Category, ScopeId, ProcessId)); + context.Send(*ServiceActorId, new TEvExecution::TEvUnregisterProcess(Category, InternalProcessId)); + } +} + +TProcessGuard::TProcessGuard(const ESpecialTaskCategory category, const TString& scopeId, const ui64 externalProcessId, + const TCPULimitsConfig& cpuLimits, const std::optional& actorId) + : Category(category) + , ScopeId(scopeId) + , ExternalProcessId(externalProcessId) + , ServiceActorId(actorId) { + if (ServiceActorId) { + NActors::TActorContext::AsActorContext().Send( + *ServiceActorId, new NConveyorComposite::TEvExecution::TEvRegisterProcess(cpuLimits, category, scopeId, InternalProcessId)); } } diff --git a/ydb/core/tx/conveyor_composite/usage/common.h b/ydb/core/tx/conveyor_composite/usage/common.h index 612b3aa511e8..e8d8619b3434 100644 --- a/ydb/core/tx/conveyor_composite/usage/common.h +++ b/ydb/core/tx/conveyor_composite/usage/common.h @@ -3,6 +3,7 @@ namespace NKikimr::NConveyorComposite { using ITask = NConveyor::ITask; +class TCPULimitsConfig; enum class ESpecialTaskCategory { Insert = 0 /* "insert" */, @@ -15,22 +16,19 @@ class TProcessGuard: TNonCopyable { private: const ESpecialTaskCategory Category; const TString ScopeId; - const ui64 ProcessId; + const ui64 ExternalProcessId; + static inline TAtomicCounter InternalCounter = 0; + const ui64 InternalProcessId = InternalCounter.Inc(); bool Finished = false; const std::optional ServiceActorId; public: - ui64 GetProcessId() const { - return ProcessId; + ui64 GetInternalProcessId() const { + return InternalProcessId; } - explicit TProcessGuard( - const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId, const std::optional& actorId) - : Category(category) - , ScopeId(scopeId) - , ProcessId(processId) - , ServiceActorId(actorId) { - } + explicit TProcessGuard(const ESpecialTaskCategory category, const TString& scopeId, const ui64 externalProcessId, + const TCPULimitsConfig& cpuLimits, const std::optional& actorId); void Finish(); diff --git a/ydb/core/tx/conveyor_composite/usage/config.cpp b/ydb/core/tx/conveyor_composite/usage/config.cpp index e0fbda75b34d..a72a3c5e72e3 100644 --- a/ydb/core/tx/conveyor_composite/usage/config.cpp +++ b/ydb/core/tx/conveyor_composite/usage/config.cpp @@ -2,12 +2,13 @@ #include +#include #include #include namespace NKikimr::NConveyorComposite::NConfig { -TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config, const ui64 usableThreadsCount) { +TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config) { if (!config.HasEnabled()) { EnabledFlag = true; } else { @@ -19,7 +20,7 @@ TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeC TWorkersPool* defWorkersPool = nullptr; WorkerPools.reserve(1 + config.GetWorkerPools().size()); if ((ui32)config.GetCategories().size() != GetEnumAllValues().size()) { - TWorkersPool wp(WorkerPools.size(), usableThreadsCount); + TWorkersPool wp(WorkerPools.size()); WorkerPools.emplace_back(std::move(wp)); defWorkersPool = &WorkerPools.front(); } @@ -35,19 +36,16 @@ TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeC } Categories[(ui64)cat.GetCategory()] = std::move(cat); } - for (auto&& i : Categories) { - if (i.IsDefault()) { - AFL_VERIFY(defWorkersPool); - AFL_VERIFY(defWorkersPool->AddLink(i.GetCategory())); - AFL_VERIFY(i.AddWorkerPool(defWorkersPool->GetWorkersPoolId())); - } - } + THashSet poolNames; for (auto&& i : config.GetWorkerPools()) { TWorkersPool wp(WorkerPools.size()); - auto conclusion = wp.DeserializeFromProto(i, usableThreadsCount); + auto conclusion = wp.DeserializeFromProto(i); if (conclusion.IsFail()) { return conclusion; } + if (!poolNames.emplace(wp.GetName()).second) { + return TConclusionStatus::Fail("pool name duplication: '" + wp.GetName() + "'"); + } WorkerPools.emplace_back(std::move(wp)); for (auto&& link : WorkerPools.back().GetLinks()) { AFL_VERIFY((ui64)link.GetCategory() < Categories.size()); @@ -57,23 +55,25 @@ TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TCompositeC } } } - for (auto&& c : Categories) { - if (c.GetWorkerPools().empty()) { - return TConclusionStatus::Fail("no worker pools for category: " + ::ToString(c.GetCategory())); + for (auto&& i : Categories) { + if (i.GetWorkerPools().empty()) { + AFL_VERIFY(defWorkersPool); + AFL_VERIFY(defWorkersPool->AddLink(i.GetCategory())); + AFL_VERIFY(i.AddWorkerPool(defWorkersPool->GetWorkersPoolId())); } } return TConclusionStatus::Success(); } -double TWorkersPool::GetWorkerCPUUsage(const ui32 workerIdx) const { - AFL_VERIFY(WorkersCountDouble); +double TWorkersPool::GetWorkerCPUUsage(const ui32 workerIdx, const ui32 totalThreadsCount) const { + const double workersCountDouble = WorkersCountInfo.GetCPUUsageDouble(totalThreadsCount); double wholePart; - const double fractionalPart = std::modf(WorkersCountDouble, &wholePart); + const double fractionalPart = std::modf(workersCountDouble, &wholePart); if (workerIdx + 1 <= wholePart) { return 1; } else { AFL_VERIFY(workerIdx == wholePart); - AFL_VERIFY(fractionalPart)("count", WorkersCountDouble); + AFL_VERIFY(fractionalPart)("count", workersCountDouble); return fractionalPart; } } @@ -101,17 +101,33 @@ TString TConfig::DebugString() const { return sb; } -TWorkersPool::TWorkersPool(const ui32 wpId, const double workersCountDouble) +TConfig TConfig::BuildDefault() { + TConfig result; + ui32 idx = 0; + for (auto&& i : GetEnumAllValues()) { + result.Categories.emplace_back(i); + result.WorkerPools.emplace_back(idx, std::nullopt, 0.33); + AFL_VERIFY(result.WorkerPools.back().AddLink(i)); + AFL_VERIFY(result.Categories.back().AddWorkerPool(idx)); + ++idx; + } + return result; +} + +TWorkersPool::TWorkersPool(const ui32 wpId, const std::optional workersCountDouble, const std::optional workersFraction) : WorkersPoolId(wpId) - , WorkersCountDouble(workersCountDouble) { - AFL_VERIFY(WorkersCountDouble); + , WorkersCountInfo(workersCountDouble, workersFraction) { + PoolName = "WP::UNDEFINED:" + ::ToString(wpId); } -TConclusionStatus TWorkersPool::DeserializeFromProto( - const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto, const ui64 usableThreadsCount) { +TConclusionStatus TWorkersPool::DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto) { if (!proto.GetLinks().size()) { return TConclusionStatus::Fail("no categories for workers pool"); } + if (proto.HasName()) { + PoolName = proto.GetName(); + } + std::set categories; for (auto&& c : proto.GetLinks()) { TWorkerPoolCategoryUsage link; auto conclusion = link.DeserializeFromProto(c); @@ -119,20 +135,19 @@ TConclusionStatus TWorkersPool::DeserializeFromProto( return conclusion; } Links.emplace_back(std::move(link)); + categories.emplace(::ToString(link.GetCategory())); + } + if (!PoolName || PoolName == "WP::DEFAULT") { + PoolName = "WP::" + JoinSeq("-", categories); } if (Links.empty()) { return TConclusionStatus::Fail("no links for workers pool"); } - if (proto.HasWorkersCount()) { - WorkersCountDouble = proto.GetWorkersCount(); - } else if (proto.HasDefaultFractionOfThreadsCount()) { - WorkersCountDouble = usableThreadsCount * proto.GetDefaultFractionOfThreadsCount(); - } else { - WorkersCountDouble = usableThreadsCount; - } - if (WorkersCountDouble <= 0) { - return TConclusionStatus::Fail( - "incorrect WorkersCount calculated: " + proto.DebugString() + " for " + ::ToString(usableThreadsCount) + " threads"); + { + auto parseConclusion = WorkersCountInfo.DeserializeFromProto(proto); + if (parseConclusion.IsFail()) { + return parseConclusion; + } } return TConclusionStatus::Success(); @@ -142,7 +157,7 @@ TString TWorkersPool::DebugString() const { TStringBuilder sb; sb << "{"; sb << "id=" << WorkersPoolId << ";"; - sb << "count=" << WorkersCountDouble << ";"; + sb << "threads=" << WorkersCountInfo.DebugString() << ";"; TStringBuilder sbLinks; sbLinks << "["; for (auto&& l : Links) { @@ -154,9 +169,13 @@ TString TWorkersPool::DebugString() const { return sb; } -ui32 TWorkersPool::GetWorkersCount() const { - AFL_VERIFY(WorkersCountDouble); - return ceil(WorkersCountDouble); +ui32 TWorkersPool::GetWorkersCount(const ui32 totalThreadsCount) const { + return WorkersCountInfo.GetThreadsCount(totalThreadsCount); +} + +const TString& TWorkersPool::GetName() const { + AFL_VERIFY(!!PoolName); + return PoolName; } TString TCategory::DebugString() const { @@ -178,6 +197,54 @@ TString TWorkerPoolCategoryUsage::DebugString() const { return sb; } +TThreadsCountInfo::TThreadsCountInfo(const std::optional count, const std::optional fraction) + : Count(count) + , Fraction(fraction) { + AFL_VERIFY(Count || Fraction); +} + +double TThreadsCountInfo::GetCPUUsageDouble(const ui32 totalThreadsCount) const { + if (Count) { + return *Count; + } + AFL_VERIFY(Fraction); + const double result = *Fraction * totalThreadsCount; + if (!result) { + return 1; + } + return result; +} + +NKikimr::TConclusionStatus TThreadsCountInfo::DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& poolInfo) { + if (poolInfo.HasWorkersCount()) { + Count = poolInfo.GetWorkersCount(); + if (*Count <= 0) { + return TConclusionStatus::Fail("incorrect threads count: " + ::ToString(*Count)); + } + Fraction.reset(); + } else if (poolInfo.HasDefaultFractionOfThreadsCount()) { + Fraction = poolInfo.GetDefaultFractionOfThreadsCount(); + if (*Fraction <= 0 || 1 < *Fraction) { + return TConclusionStatus::Fail("incorrect threads count fraction: " + ::ToString(*Fraction)); + } + Count.reset(); + } + return TConclusionStatus::Success(); +} + +TString TThreadsCountInfo::DebugString() const { + TStringBuilder sb; + sb << "{"; + if (Count) { + sb << "c=" << *Count << ";"; + } + if (Fraction) { + sb << "f=" << *Fraction << ";"; + } + sb << "}"; + return sb; +} + } // namespace NKikimr::NConveyorComposite::NConfig namespace NKikimr::NConveyorComposite { diff --git a/ydb/core/tx/conveyor_composite/usage/config.h b/ydb/core/tx/conveyor_composite/usage/config.h index 2f074b24bb2f..3940953e9708 100644 --- a/ydb/core/tx/conveyor_composite/usage/config.h +++ b/ydb/core/tx/conveyor_composite/usage/config.h @@ -6,6 +6,7 @@ #include #include +#include #include namespace NKikimr::NConveyorComposite::NConfig { @@ -38,15 +39,38 @@ class TWorkerPoolCategoryUsage { } }; +class TThreadsCountInfo { +private: + YDB_READONLY_DEF(std::optional, Count); + YDB_READONLY(std::optional, Fraction, 0.33); + +public: + TThreadsCountInfo() = default; + TThreadsCountInfo(const std::optional count, const std::optional fraction); + + TString DebugString() const; + + ui32 GetThreadsCount(const ui32 totalThreadsCount) const { + return std::ceil(GetCPUUsageDouble(totalThreadsCount)); + } + + double GetCPUUsageDouble(const ui32 totalThreadsCount) const; + + TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& poolInfo); +}; + class TWorkersPool { private: + TString PoolName; YDB_READONLY(ui32, WorkersPoolId, 0); - YDB_READONLY(double, WorkersCountDouble, 0); + YDB_READONLY_DEF(TThreadsCountInfo, WorkersCountInfo); YDB_READONLY_DEF(std::vector, Links); public: - double GetWorkerCPUUsage(const ui32 workerIdx) const; - ui32 GetWorkersCount() const; + const TString& GetName() const; + + double GetWorkerCPUUsage(const ui32 workerIdx, const ui32 totalThreadsCount) const; + ui32 GetWorkersCount(const ui32 totalThreadsCount) const; bool AddLink(const ESpecialTaskCategory cat) { for (auto&& i : Links) { @@ -62,12 +86,12 @@ class TWorkersPool { TWorkersPool(const ui32 wpId) : WorkersPoolId(wpId) { + PoolName = "WP::DEFAULT"; } - TWorkersPool(const ui32 wpId, const double workersCountDouble); + TWorkersPool(const ui32 wpId, const std::optional workersCountDouble, const std::optional workersFraction); - [[nodiscard]] TConclusionStatus DeserializeFromProto( - const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto, const ui64 usableThreadsCount); + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig::TWorkersPool& proto); }; class TCategory { @@ -75,7 +99,6 @@ class TCategory { YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); YDB_READONLY(ui32, QueueSizeLimit, 256 * 1024); YDB_READONLY_DEF(std::vector, WorkerPools); - YDB_READONLY_FLAG(Default, true); public: TString DebugString() const; @@ -97,7 +120,6 @@ class TCategory { if (proto.HasQueueSizeLimit()) { QueueSizeLimit = proto.GetQueueSizeLimit(); } - DefaultFlag = false; return TConclusionStatus::Success(); } @@ -112,10 +134,22 @@ class TConfig { YDB_READONLY_DEF(std::vector, WorkerPools); YDB_READONLY_FLAG(Enabled, true); + TConfig() = default; + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config); + public: - const TCategory& GetCategoryConfig(const ESpecialTaskCategory cat) const; + static TConfig BuildDefault(); - [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TCompositeConveyorConfig& config, const ui64 usableThreadsCount); + static TConclusion BuildFromProto(const NKikimrConfig::TCompositeConveyorConfig& protoConfig) { + TConfig config; + auto conclusion = config.DeserializeFromProto(protoConfig); + if (conclusion.IsFail()) { + return conclusion; + } + return config; + } + + const TCategory& GetCategoryConfig(const ESpecialTaskCategory cat) const; TString DebugString() const; }; @@ -135,4 +169,4 @@ class TCPULimitsConfig { TString DebugString() const; }; -} +} // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/usage/events.cpp b/ydb/core/tx/conveyor_composite/usage/events.cpp index 5c94bd68be2c..5b7d162f5a76 100644 --- a/ydb/core/tx/conveyor_composite/usage/events.cpp +++ b/ydb/core/tx/conveyor_composite/usage/events.cpp @@ -4,11 +4,10 @@ namespace NKikimr::NConveyorComposite { -TEvExecution::TEvNewTask::TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) +TEvExecution::TEvNewTask::TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const ui64 internalProcessId) : Task(task) , Category(category) - , ScopeId(scopeId) - , ProcessId(processId) { + , InternalProcessId(internalProcessId) { AFL_VERIFY(Task); } diff --git a/ydb/core/tx/conveyor_composite/usage/events.h b/ydb/core/tx/conveyor_composite/usage/events.h index a3169dad9bd1..ae8891069d99 100644 --- a/ydb/core/tx/conveyor_composite/usage/events.h +++ b/ydb/core/tx/conveyor_composite/usage/events.h @@ -22,31 +22,30 @@ struct TEvExecution { class TEvNewTask: public NActors::TEventLocal { private: - YDB_READONLY_DEF(ITask::TPtr, Task); + YDB_ACCESSOR_DEF(ITask::TPtr, Task); YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); - YDB_READONLY_DEF(TString, ScopeId); - YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, InternalProcessId, 0); YDB_READONLY(TMonotonic, ConstructInstant, TMonotonic::Now()); public: TEvNewTask() = default; - explicit TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId); + explicit TEvNewTask(ITask::TPtr task, const ESpecialTaskCategory category, const ui64 processId); }; class TEvRegisterProcess: public NActors::TEventLocal { private: YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); YDB_READONLY_DEF(TString, ScopeId); - YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, InternalProcessId, 0); YDB_READONLY_DEF(TCPULimitsConfig, CPULimits); public: explicit TEvRegisterProcess( - const TCPULimitsConfig& cpuLimits, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + const TCPULimitsConfig& cpuLimits, const ESpecialTaskCategory category, const TString& scopeId, const ui64 internalProcessId) : Category(category) , ScopeId(scopeId) - , ProcessId(processId) + , InternalProcessId(internalProcessId) , CPULimits(cpuLimits) { } }; @@ -54,14 +53,12 @@ struct TEvExecution { class TEvUnregisterProcess: public NActors::TEventLocal { private: YDB_READONLY(ESpecialTaskCategory, Category, ESpecialTaskCategory::Insert); - YDB_READONLY_DEF(TString, ScopeId); - YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, InternalProcessId, 0); public: - explicit TEvUnregisterProcess(const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) + explicit TEvUnregisterProcess(const ESpecialTaskCategory category, const ui64 internalProcessId) : Category(category) - , ScopeId(scopeId) - , ProcessId(processId) { + , InternalProcessId(internalProcessId) { } }; }; diff --git a/ydb/core/tx/conveyor_composite/usage/service.h b/ydb/core/tx/conveyor_composite/usage/service.h index 336b5b5b9da0..42a72d63f9ae 100644 --- a/ydb/core/tx/conveyor_composite/usage/service.h +++ b/ydb/core/tx/conveyor_composite/usage/service.h @@ -9,25 +9,21 @@ #include namespace NKikimr::NConveyorComposite { -template -class TServiceOperatorImpl { + +class TServiceOperator { private: - using TSelf = TServiceOperatorImpl; + using TSelf = TServiceOperator; std::atomic IsEnabledFlag = false; static void Register(const NConfig::TConfig& serviceConfig) { Singleton()->IsEnabledFlag = serviceConfig.IsEnabled(); } - static const TString& GetConveyorName() { - Y_ABORT_UNLESS(TConveyorPolicy::Name.size() == 4); - return TConveyorPolicy::Name; - } public: - static bool SendTaskToExecute(const std::shared_ptr& task, const ESpecialTaskCategory category, const TString& scopeId, const ui64 processId) { + static bool SendTaskToExecute(const std::shared_ptr& task, const ESpecialTaskCategory category, const ui64 internalProcessId) { if (TSelf::IsEnabled() && NActors::TlsActivationContext) { auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; - context.Send(MakeServiceId(selfId.NodeId()), new NConveyorComposite::TEvExecution::TEvNewTask(task, category, scopeId, processId)); + context.Send(MakeServiceId(selfId.NodeId()), new NConveyorComposite::TEvExecution::TEvNewTask(task, category, internalProcessId)); return true; } else { task->Execute(nullptr, task); @@ -38,46 +34,54 @@ class TServiceOperatorImpl { return Singleton()->IsEnabledFlag; } static NActors::TActorId MakeServiceId(const ui32 nodeId) { - return NActors::TActorId(nodeId, "SrvcConv" + GetConveyorName()); + return NActors::TActorId(nodeId, "SrvcConvCmps"); } static NActors::IActor* CreateService(const NConfig::TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) { Register(config); - return new TDistributor(config, GetConveyorName(), conveyorSignals); + return new TDistributor(config, conveyorSignals); } static TProcessGuard StartProcess( const ESpecialTaskCategory category, const TString& scopeId, const ui64 externalProcessId, const TCPULimitsConfig& cpuLimits) { if (TSelf::IsEnabled() && NActors::TlsActivationContext) { auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; - context.Send(MakeServiceId(selfId.NodeId()), - new NConveyorComposite::TEvExecution::TEvRegisterProcess(cpuLimits, category, scopeId, externalProcessId)); - return TProcessGuard(category, scopeId, externalProcessId, MakeServiceId(selfId.NodeId())); + return TProcessGuard(category, scopeId, externalProcessId, cpuLimits, MakeServiceId(selfId.NodeId())); } else { - return TProcessGuard(category, scopeId, externalProcessId, {}); + return TProcessGuard(category, scopeId, externalProcessId, cpuLimits, {}); } } }; -class TScanConveyorPolicy { +class TInsertServiceOperator { public: - static const inline TString Name = "Scan"; - static constexpr bool EnableProcesses = true; + static bool SendTaskToExecute(const std::shared_ptr& task) { + return TServiceOperator::SendTaskToExecute(task, ESpecialTaskCategory::Insert, 0); + } }; -class TCompConveyorPolicy { +class TNormalizerServiceOperator { public: - static const inline TString Name = "Comp"; - static constexpr bool EnableProcesses = false; + static bool SendTaskToExecute(const std::shared_ptr& task) { + return TServiceOperator::SendTaskToExecute(task, ESpecialTaskCategory::Normalizer, 0); + } }; -class TInsertConveyorPolicy { +class TCompServiceOperator { public: - static const inline TString Name = "Isrt"; - static constexpr bool EnableProcesses = false; + static bool SendTaskToExecute(const std::shared_ptr& task) { + return TServiceOperator::SendTaskToExecute(task, ESpecialTaskCategory::Compaction, 0); + } }; -using TScanServiceOperator = TServiceOperatorImpl; -using TCompServiceOperator = TServiceOperatorImpl; -using TInsertServiceOperator = TServiceOperatorImpl; +class TScanServiceOperator { +public: + static bool SendTaskToExecute(const std::shared_ptr& task, const ui64 internalProcessId) { + return TServiceOperator::SendTaskToExecute(task, ESpecialTaskCategory::Scan, internalProcessId); + } + + static TProcessGuard StartProcess(const ui64 externalProcessId, const TString& scopeId, const TCPULimitsConfig& cpuLimits) { + return TServiceOperator::StartProcess(ESpecialTaskCategory::Scan, scopeId, externalProcessId, cpuLimits); + } +}; } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp b/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp index 046394f3c71c..f77728e255de 100644 --- a/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp +++ b/ydb/core/tx/conveyor_composite/ut/ut_simple.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -105,14 +106,14 @@ class TSimpleRequest: public IRequestProcessor { } virtual void DoAddTask(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId) override { actorSystem.Send(distributorId, - new TEvExecution::TEvNewTask(std::make_shared(TDuration::MicroSeconds(40), Counter), Category, ScopeId, ProcessId)); + new TEvExecution::TEvNewTask(std::make_shared(TDuration::MicroSeconds(40), Counter), Category, ProcessId)); CounterTasks.Inc(); } virtual bool DoCheckFinished() override { return CounterTasks.Val() == Counter.Val(); } virtual void DoFinish(NActors::TActorSystem& actorSystem, const NActors::TActorId distributorId, const TDuration /*d*/) override { - actorSystem.Send(distributorId, new TEvExecution::TEvUnregisterProcess(Category, ScopeId, ProcessId)); + actorSystem.Send(distributorId, new TEvExecution::TEvUnregisterProcess(Category, ProcessId)); } public: @@ -131,6 +132,7 @@ class TTestingExecutor { virtual ui32 GetTasksCount() const { return 1000000; } + public: virtual double GetThreadsCount() const { return 9.5; @@ -147,9 +149,8 @@ class TTestingExecutor { NKikimrConfig::TCompositeConveyorConfig protoConfig; AFL_VERIFY(google::protobuf::TextFormat::ParseFromString(textProto, &protoConfig)); - NConfig::TConfig config; - config.DeserializeFromProto(protoConfig, threadsCount).Validate(); - const auto actorId = actorSystem.Register(TCompServiceOperator::CreateService(config, counters)); + NConfig::TConfig config = NConfig::TConfig::BuildFromProto(protoConfig).DetachResult(); + const auto actorId = actorSystem.Register(TServiceOperator::CreateService(config, counters)); std::vector> requests = GetRequests(); for (auto&& i : requests) { @@ -191,14 +192,15 @@ class TTestingExecutor { ++idx; } } - Cerr << (GetThreadsCount() * (TMonotonic::Now() - globalStart) / (1.0 * requests.size() * GetTasksCount())).MicroSeconds() << "us per task" - << Endl; + Cerr << (GetThreadsCount() * (TMonotonic::Now() - globalStart) / (1.0 * requests.size() * GetTasksCount())).MicroSeconds() + << "us per task" << Endl; TStringBuilder sb; for (auto&& i : durations) { sb << i << ";"; } Cerr << sb << Endl; - Sleep(TDuration::Seconds(5)); + Sleep(TDuration::Seconds(15)); + AFL_VERIFY(NKikimr::NColumnShard::TMonitoringObjectsCounter::GetCounter().Val() == 4)("count", NKikimr::NColumnShard::TMonitoringObjectsCounter::GetCounter().Val()); actorSystem.Stop(); actorSystem.Cleanup(); @@ -347,8 +349,8 @@ Y_UNIT_TEST_SUITE(CompositeConveyorTests) { } virtual std::vector> GetRequests() override { return { std::make_shared("1", ESpecialTaskCategory::Insert, "1", 1), - std::make_shared("2", ESpecialTaskCategory::Insert, "2", 1), - std::make_shared("3", ESpecialTaskCategory::Insert, "3", 1) }; + std::make_shared("2", ESpecialTaskCategory::Insert, "2", 2), + std::make_shared("3", ESpecialTaskCategory::Insert, "3", 3) }; } public: @@ -395,26 +397,24 @@ Y_UNIT_TEST_SUITE(CompositeConveyorTests) { GetThreadsCount()); } virtual std::vector> GetRequests() override { - return { - std::make_shared("I_1_1", ESpecialTaskCategory::Insert, "1", 1), - std::make_shared("I_2_1", ESpecialTaskCategory::Insert, "2", 1), - std::make_shared("I_3_1", ESpecialTaskCategory::Insert, "3", 1), - std::make_shared("S_1_1", ESpecialTaskCategory::Scan, "1", 1), - std::make_shared("S_2_1", ESpecialTaskCategory::Scan, "2", 1), - std::make_shared("S_3_1", ESpecialTaskCategory::Scan, "3", 1), - std::make_shared("N_1_1", ESpecialTaskCategory::Normalizer, "1", 1), - std::make_shared("N_2_1", ESpecialTaskCategory::Normalizer, "2", 1), - std::make_shared("N_3_1", ESpecialTaskCategory::Normalizer, "3", 1), - std::make_shared("I_1_2", ESpecialTaskCategory::Insert, "1", 2), - std::make_shared("I_2_2", ESpecialTaskCategory::Insert, "2", 2), - std::make_shared("I_3_2", ESpecialTaskCategory::Insert, "3", 2), - std::make_shared("S_1_2", ESpecialTaskCategory::Scan, "1", 2), - std::make_shared("S_2_2", ESpecialTaskCategory::Scan, "2", 2), - std::make_shared("S_3_2", ESpecialTaskCategory::Scan, "3", 2), - std::make_shared("N_1_2", ESpecialTaskCategory::Normalizer, "1", 2), - std::make_shared("N_2_2", ESpecialTaskCategory::Normalizer, "2", 2), - std::make_shared("N_3_2", ESpecialTaskCategory::Normalizer, "3", 2) - }; + return { std::make_shared("I_1_1", ESpecialTaskCategory::Insert, "1", 1), + std::make_shared("I_2_1", ESpecialTaskCategory::Insert, "2", 2), + std::make_shared("I_3_1", ESpecialTaskCategory::Insert, "3", 3), + std::make_shared("S_1_1", ESpecialTaskCategory::Scan, "1", 4), + std::make_shared("S_2_1", ESpecialTaskCategory::Scan, "2", 5), + std::make_shared("S_3_1", ESpecialTaskCategory::Scan, "3", 6), + std::make_shared("N_1_1", ESpecialTaskCategory::Normalizer, "1", 7), + std::make_shared("N_2_1", ESpecialTaskCategory::Normalizer, "2", 8), + std::make_shared("N_3_1", ESpecialTaskCategory::Normalizer, "3", 9), + std::make_shared("I_1_2", ESpecialTaskCategory::Insert, "1", 21), + std::make_shared("I_2_2", ESpecialTaskCategory::Insert, "2", 22), + std::make_shared("I_3_2", ESpecialTaskCategory::Insert, "3", 23), + std::make_shared("S_1_2", ESpecialTaskCategory::Scan, "1", 24), + std::make_shared("S_2_2", ESpecialTaskCategory::Scan, "2", 25), + std::make_shared("S_3_2", ESpecialTaskCategory::Scan, "3", 26), + std::make_shared("N_1_2", ESpecialTaskCategory::Normalizer, "1", 27), + std::make_shared("N_2_2", ESpecialTaskCategory::Normalizer, "2", 28), + std::make_shared("N_3_2", ESpecialTaskCategory::Normalizer, "3", 29) }; } public: diff --git a/ydb/library/signals/owner.cpp b/ydb/library/signals/owner.cpp index c97a66d6e5eb..c2f000dae124 100644 --- a/ydb/library/signals/owner.cpp +++ b/ydb/library/signals/owner.cpp @@ -27,8 +27,8 @@ TCommonCountersOwner::TCommonCountersOwner(const TString& module, TIntrusivePtr< { if (baseSignals) { SubGroup = baseSignals->GetSubgroup("module_id", module); - } else if (NActors::TlsActivationContext) { - SubGroup = GetServiceCounters(AppData()->Counters, "tablets")->GetSubgroup("subsystem", "columnshard")->GetSubgroup("module_id", module); + } else if (NActors::TlsActivationContext && HasAppData()) { + SubGroup = GetServiceCounters(AppDataVerified().Counters, "tablets")->GetSubgroup("subsystem", "columnshard")->GetSubgroup("module_id", module); } else { SubGroup = new NMonitoring::TDynamicCounters(); } From 3c67670bbe797ea58f9f0a94cf04e44c62562d5b Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sun, 15 Jun 2025 08:23:14 +0300 Subject: [PATCH 08/62] Fix categories usage with scope limits (#19665) --- .../conveyor_composite/service/category.cpp | 22 ++++++++++- .../tx/conveyor_composite/service/category.h | 11 +----- .../service/workers_pool.cpp | 39 ++++++++++--------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/ydb/core/tx/conveyor_composite/service/category.cpp b/ydb/core/tx/conveyor_composite/service/category.cpp index 0f2234794e45..e56a28103930 100644 --- a/ydb/core/tx/conveyor_composite/service/category.cpp +++ b/ydb/core/tx/conveyor_composite/service/category.cpp @@ -13,7 +13,7 @@ void TProcessCategory::DoQuant(const TMonotonic newStart) { } } -TWorkerTask TProcessCategory::ExtractTaskWithPrediction(const std::shared_ptr& counters) { +std::optional TProcessCategory::ExtractTaskWithPrediction(const std::shared_ptr& counters, THashSet& scopeIds) { std::shared_ptr pMin; TDuration dMin; for (auto&& [_, p] : ProcessesWithTasks) { @@ -26,10 +26,16 @@ TWorkerTask TProcessCategory::ExtractTaskWithPrediction(const std::shared_ptrExtractTaskWithPrediction(counters); if (pMin->GetTasksCount() == 0) { AFL_VERIFY(ProcessesWithTasks.erase(pMin->GetProcessId())); + } + if (scopeIds.emplace(pMin->GetScope()->GetScopeId()).second) { + pMin->GetScope()->IncInFlight(); } Counters->WaitingQueueSize->Set(WaitingTasksCount->Val()); return result; @@ -58,7 +64,7 @@ TProcessScope* TProcessCategory::MutableProcessScopeOptional(const TString& scop std::shared_ptr TProcessCategory::RegisterScope(const TString& scopeId, const TCPULimitsConfig& processCpuLimits) { TCPUGroup::TPtr cpuGroup = std::make_shared(processCpuLimits.GetCPUGroupThreadsLimitDef(256)); - auto info = Scopes.emplace(scopeId, std::make_shared(scopeId, std::move(cpuGroup), std::move(CPUUsage))); + auto info = Scopes.emplace(scopeId, std::make_shared(scopeId, std::move(cpuGroup), CPUUsage)); AFL_VERIFY(info.second); return info.first->second; } @@ -83,4 +89,16 @@ void TProcessCategory::UnregisterScope(const TString& name) { Scopes.erase(it); } +void TProcessCategory::PutTaskResult(TWorkerTaskResult&& result, THashSet& scopeIds) { + const ui64 internalProcessId = result.GetProcessId(); + auto it = Processes.find(internalProcessId); + if (scopeIds.emplace(result.GetScope()->GetScopeId()).second) { + result.GetScope()->DecInFlight(); + } + if (it == Processes.end()) { + return; + } + it->second->PutTaskResult(std::move(result)); +} + } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/category.h b/ydb/core/tx/conveyor_composite/service/category.h index 1a78708b6eb0..94261a68170f 100644 --- a/ydb/core/tx/conveyor_composite/service/category.h +++ b/ydb/core/tx/conveyor_composite/service/category.h @@ -45,14 +45,7 @@ class TProcessCategory: public TNonCopyable { } } - void PutTaskResult(TWorkerTaskResult&& result) { - const ui64 internalProcessId = result.GetProcessId(); - auto it = Processes.find(internalProcessId); - if (it == Processes.end()) { - return; - } - it->second->PutTaskResult(std::move(result)); - } + void PutTaskResult(TWorkerTaskResult&& result, THashSet& scopeIds); void RegisterProcess(const ui64 internalProcessId, std::shared_ptr&& scope) { scope->IncProcesses(); @@ -75,7 +68,7 @@ class TProcessCategory: public TNonCopyable { bool HasTasks() const; void DoQuant(const TMonotonic newStart); - TWorkerTask ExtractTaskWithPrediction(const std::shared_ptr& counters); + std::optional ExtractTaskWithPrediction(const std::shared_ptr& counters, THashSet& scopeIds); TProcessScope& MutableProcessScope(const TString& scopeName); TProcessScope* MutableProcessScopeOptional(const TString& scopeName); std::shared_ptr GetProcessScopePtrVerified(const TString& scopeName) const; diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp index 099eec259ecd..d7f36ce15db1 100644 --- a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp @@ -64,26 +64,30 @@ bool TWorkersPool::DrainTasks() { return r.GetCPUUsage()->CalcWeight(r.GetWeight()) < l.GetCPUUsage()->CalcWeight(l.GetWeight()); }; std::make_heap(Processes.begin(), Processes.end(), predHeap); - AFL_VERIFY(Processes.size()); + std::vector procLocal = Processes; + AFL_VERIFY(procLocal.size()); bool newTask = false; - while (ActiveWorkersIdx.size() && Processes.front().GetCategory()->HasTasks()) { + while (ActiveWorkersIdx.size() && procLocal.size() && procLocal.front().GetCategory()->HasTasks()) { TDuration predicted = TDuration::Zero(); std::vector tasks; - THashMap> scopes; - while ((tasks.empty() || predicted < DeliveringDuration.GetValue() * 10) && Processes.front().GetCategory()->HasTasks()) { - std::pop_heap(Processes.begin(), Processes.end(), predHeap); - tasks.emplace_back(Processes.back().GetCategory()->ExtractTaskWithPrediction(Processes.back().GetCounters())); - Processes.back().GetCPUUsage()->AddPredicted(tasks.back().GetPredictedDuration()); + THashSet scopes; + while (procLocal.size() && (tasks.empty() || predicted < DeliveringDuration.GetValue() * 10) && + procLocal.front().GetCategory()->HasTasks()) { + std::pop_heap(procLocal.begin(), procLocal.end(), predHeap); + auto task = procLocal.back().GetCategory()->ExtractTaskWithPrediction(procLocal.back().GetCounters(), scopes); + if (!task) { + procLocal.pop_back(); + continue; + } + tasks.emplace_back(std::move(*task)); + procLocal.back().GetCPUUsage()->AddPredicted(tasks.back().GetPredictedDuration()); predicted += tasks.back().GetPredictedDuration(); - std::push_heap(Processes.begin(), Processes.end(), predHeap); - scopes.emplace(tasks.back().GetScope()->GetScopeId(), tasks.back().GetScope()); - } - for (auto&& i : scopes) { - i.second->IncInFlight(); + std::push_heap(procLocal.begin(), procLocal.end(), predHeap); } newTask = true; - AFL_VERIFY(tasks.size()); - RunTask(std::move(tasks)); + if (tasks.size()) { + RunTask(std::move(tasks)); + } } if (!Processes.front().GetCategory()->HasTasks()) { Counters->NoTasks->Add(1); @@ -93,7 +97,7 @@ bool TWorkersPool::DrainTasks() { void TWorkersPool::PutTaskResults(std::vector&& result) { // const ui32 catIdx = (ui32)result.GetCategory(); - std::set scopeIds; + THashSet scopeIds; for (auto&& t : result) { bool found = false; for (auto&& i : Processes) { @@ -103,10 +107,7 @@ void TWorkersPool::PutTaskResults(std::vector&& result) { i.GetCounters()->ExecuteDuration->Add((t.GetFinish() - t.GetStart()).MicroSeconds()); found = true; i.GetCPUUsage()->Exchange(t.GetPredictedDuration(), t.GetStart(), t.GetFinish()); - if (scopeIds.emplace(t.GetScope()->GetScopeId()).second) { - t.GetScope()->DecInFlight(); - } - i.GetCategory()->PutTaskResult(std::move(t)); + i.GetCategory()->PutTaskResult(std::move(t), scopeIds); break; } } From 49e482e95b4879a83477e6578d98833803ea0d34 Mon Sep 17 00:00:00 2001 From: Semyon Date: Tue, 10 Jun 2025 16:07:30 +0300 Subject: [PATCH 09/62] filter iterator corrections (#19359) --- ydb/core/formats/arrow/arrow_filter.cpp | 13 +++++++++---- ydb/core/formats/arrow/arrow_filter.h | 13 ++++++++++--- ydb/core/formats/arrow/reader/batch_iterator.h | 6 ++---- ydb/core/tx/columnshard/common/snapshot.h | 2 +- .../common_reader/iterator/default_fetching.h | 2 +- .../common_reader/iterator/sub_columns_fetching.h | 2 +- .../engines/reader/plain_reader/iterator/source.cpp | 2 +- .../reader/simple_reader/duplicates/merge.cpp | 4 ++++ .../reader/simple_reader/iterator/source.cpp | 2 +- .../simple_reader/iterator/sync_points/limit.h | 2 +- 10 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ydb/core/formats/arrow/arrow_filter.cpp b/ydb/core/formats/arrow/arrow_filter.cpp index f7544701c2ca..69ef49afb759 100644 --- a/ydb/core/formats/arrow/arrow_filter.cpp +++ b/ydb/core/formats/arrow/arrow_filter.cpp @@ -518,14 +518,19 @@ TColumnFilter TColumnFilter::CombineSequentialAnd(const TColumnFilter& extFilter } } -TColumnFilter::TIterator TColumnFilter::GetIterator(const bool reverse, const ui32 expectedSize) const { +TColumnFilter::TIterator TColumnFilter::GetBegin(const bool reverse, const ui32 expectedSize) const { + return GetIterator(reverse, expectedSize, 0); +} + +TColumnFilter::TIterator TColumnFilter::GetIterator(const bool reverse, const ui32 expectedSize, const ui64 startOffset) const { + AFL_VERIFY(expectedSize >= startOffset); if (IsTotalAllowFilter()) { - return TIterator(reverse, expectedSize, true); + return TIterator(reverse, expectedSize, true, startOffset); } else if (IsTotalDenyFilter()) { - return TIterator(reverse, expectedSize, false); + return TIterator(reverse, expectedSize, false, startOffset); } else { AFL_VERIFY(expectedSize == GetRecordsCountVerified())("expected", expectedSize)("count", GetRecordsCountVerified())("reverse", reverse); - return TIterator(reverse, Filter, GetStartValue(reverse)); + return TIterator(reverse, Filter, GetStartValue(reverse), startOffset); } } diff --git a/ydb/core/formats/arrow/arrow_filter.h b/ydb/core/formats/arrow/arrow_filter.h index 4fae08842234..64a5845f73f9 100644 --- a/ydb/core/formats/arrow/arrow_filter.h +++ b/ydb/core/formats/arrow/arrow_filter.h @@ -150,7 +150,7 @@ class TColumnFilter { public: TString DebugString() const; - TIterator(const bool reverse, const std::vector& filter, const bool startValue) + TIterator(const bool reverse, const std::vector& filter, const bool startValue, const ui64 startOffset) : FilterPointer(&filter) , CurrentValue(startValue) , FinishPosition(reverse ? -1 : FilterPointer->size()) @@ -163,9 +163,12 @@ class TColumnFilter { } CurrentRemainVolume = (*FilterPointer)[Position]; } + if (startOffset) { + Next(startOffset); + } } - TIterator(const bool reverse, const ui32 size, const bool startValue) + TIterator(const bool reverse, const ui32 size, const bool startValue, const ui64 startOffset) : CurrentValue(startValue) , FinishPosition(reverse ? -1 : 1) , DeltaPosition(reverse ? -1 : 1) { @@ -177,6 +180,9 @@ class TColumnFilter { } CurrentRemainVolume = size; } + if (startOffset) { + Next(startOffset); + } } bool GetCurrentAcceptance() const { @@ -194,7 +200,8 @@ class TColumnFilter { TString DebugString() const; - TIterator GetIterator(const bool reverse, const ui32 expectedSize) const; + TIterator GetBegin(const bool reverse, const ui32 expectedSize) const; + TIterator GetIterator(const bool reverse, const ui32 expectedSize, const ui64 startOffset) const; bool CheckSlice(const ui32 offset, const ui32 count) const; diff --git a/ydb/core/formats/arrow/reader/batch_iterator.h b/ydb/core/formats/arrow/reader/batch_iterator.h index 5fba3f53fc50..4fce2e46970e 100644 --- a/ydb/core/formats/arrow/reader/batch_iterator.h +++ b/ydb/core/formats/arrow/reader/batch_iterator.h @@ -87,10 +87,8 @@ class TBatchIterator { Y_ABORT_UNLESS(KeyColumns.InitPosition(GetPosition(order.GetStart()))); Y_ABORT_UNLESS(VersionColumns.InitPosition(GetPosition(order.GetStart()))); if (Filter) { - FilterIterator = std::make_shared(Filter->GetIterator(order.GetIsReversed(), RecordsCount)); - if (order.GetStart()) { - FilterIterator->Next(order.GetStart()); - } + FilterIterator = + std::make_shared(Filter->GetIterator(order.GetIsReversed(), RecordsCount, order.GetStart())); } } diff --git a/ydb/core/tx/columnshard/common/snapshot.h b/ydb/core/tx/columnshard/common/snapshot.h index 7e1b7a78b148..bed4b5da80d3 100644 --- a/ydb/core/tx/columnshard/common/snapshot.h +++ b/ydb/core/tx/columnshard/common/snapshot.h @@ -103,7 +103,7 @@ class TSnapshot { TString DebugString() const; NJson::TJsonValue DebugJson() const; - operator size_t() const { + explicit operator size_t() const { return CombineHashes(PlanStep, TxId); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h index 4a0cc71cb5f3..bee2026acca6 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/default_fetching.h @@ -90,7 +90,7 @@ class TDefaultFetchLogic: public IKernelFetchLogic { auto reading = blobsAction.GetReading(*StorageId); auto filterPtr = source->GetStageData().GetAppliedFilter(); const NArrow::TColumnFilter& cFilter = filterPtr ? *filterPtr : NArrow::TColumnFilter::BuildAllowFilter(); - auto itFilter = cFilter.GetIterator(false, source->GetRecordsCount()); + auto itFilter = cFilter.GetBegin(false, source->GetRecordsCount()); bool itFinished = false; for (auto&& c : columnChunks) { AFL_VERIFY(!itFinished); diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/sub_columns_fetching.h b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/sub_columns_fetching.h index 57cb35904c8d..61b2758d92a4 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/sub_columns_fetching.h +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/sub_columns_fetching.h @@ -237,7 +237,7 @@ class TSubColumnsFetchLogic: public IKernelFetchLogic { reading->SetIsBackgroundProcess(false); auto filterPtr = source->GetStageData().GetAppliedFilter(); const NArrow::TColumnFilter& cFilter = filterPtr ? *filterPtr : NArrow::TColumnFilter::BuildAllowFilter(); - auto itFilter = cFilter.GetIterator(false, source->GetRecordsCount()); + auto itFilter = cFilter.GetBegin(false, source->GetRecordsCount()); bool itFinished = false; auto accessor = context.GetAccessors().GetAccessorOptional(GetEntityId()); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 0aa21c4d7f55..e09c63ef9988 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -87,7 +87,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob if (columnChunks.empty()) { continue; } - auto itFilter = cFilter.GetIterator(false, Portion->GetRecordsCount()); + auto itFilter = cFilter.GetBegin(false, Portion->GetRecordsCount()); bool itFinished = false; for (auto&& c : columnChunks) { AFL_VERIFY(!itFinished); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp index 59ef376ebb4f..07a4ff2bfd51 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/duplicates/merge.cpp @@ -9,6 +9,7 @@ class TFiltersBuilder { THashMap Filters; YDB_READONLY(ui64, RowsAdded, 0); YDB_READONLY(ui64, RowsSkipped, 0); + bool IsDone = false; void AddImpl(const ui64 sourceId, const bool value) { auto* findFilter = Filters.FindPtr(sourceId); @@ -35,10 +36,13 @@ class TFiltersBuilder { } THashMap&& ExtractFilters() && { + AFL_VERIFY(!IsDone); + IsDone = true; return std::move(Filters); } void AddSource(const ui64 sourceId) { + AFL_VERIFY(!IsDone); AFL_VERIFY(Filters.emplace(sourceId, NArrow::TColumnFilter::BuildAllowFilter()).second); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index 43f0117d65e6..e302537fd4ff 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -105,7 +105,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob if (columnChunks.empty()) { continue; } - auto itFilter = cFilter.GetIterator(false, Portion->GetRecordsCount()); + auto itFilter = cFilter.GetBegin(false, Portion->GetRecordsCount()); bool itFinished = false; for (auto&& c : columnChunks) { AFL_VERIFY(!itFinished); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sync_points/limit.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sync_points/limit.h index e829f6eb648b..befd50f49862 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sync_points/limit.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/sync_points/limit.h @@ -73,7 +73,7 @@ class TSyncPointLimitControl: public ISyncPoint { , Filter(filter ? filter : std::make_shared(NArrow::TColumnFilter::BuildAllowFilter())) { AFL_VERIFY(arrs.size()); AFL_VERIFY(arrs.front()->GetRecordsCount()); - FilterIterator = std::make_shared(Filter->GetIterator(Reverse, arrs.front()->GetRecordsCount())); + FilterIterator = std::make_shared(Filter->GetBegin(Reverse, arrs.front()->GetRecordsCount())); auto prefixSchema = Source->GetSourceSchema()->GetIndexInfo().GetReplaceKeyPrefix(arrs.size()); auto copyArrs = arrs; auto batch = std::make_shared(prefixSchema->fields(), std::move(copyArrs)); From a0592966a392867c36c140bf42550a447144027b Mon Sep 17 00:00:00 2001 From: Semyon Date: Tue, 10 Jun 2025 16:07:46 +0300 Subject: [PATCH 10/62] aborted scan counters (#19353) --- ydb/core/tx/columnshard/counters/scan.h | 10 ++++++++-- .../columnshard/engines/reader/abstract/read_context.h | 2 +- ydb/core/tx/columnshard/engines/reader/actor/actor.cpp | 3 +-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/columnshard/counters/scan.h b/ydb/core/tx/columnshard/counters/scan.h index 897610fbbdba..f395e40f43d1 100644 --- a/ydb/core/tx/columnshard/counters/scan.h +++ b/ydb/core/tx/columnshard/counters/scan.h @@ -361,6 +361,7 @@ class TConcreteScanCounters: public TScanCounters { std::shared_ptr ResultsForSourceCount = std::make_shared(); std::shared_ptr ResultsForReplyGuard = std::make_shared(); std::shared_ptr FilterFetchingGuard = std::make_shared(); + std::shared_ptr AbortsGuard = std::make_shared(); std::shared_ptr TotalExecutionDurationUs = std::make_shared(); THashMap> SkipNodesCount; THashMap> ExecuteNodesCount; @@ -401,7 +402,8 @@ class TConcreteScanCounters: public TScanCounters { << "ResourcesAllocationTasksCount:" << ResourcesAllocationTasksCount->Val() << ";" << "ResultsForSourceCount:" << ResultsForSourceCount->Val() << ";" << "ResultsForReplyGuard:" << ResultsForReplyGuard->Val() << ";" - << "FilterFetchingGuard:" << FilterFetchingGuard->Val() << ";"; + << "FilterFetchingGuard:" << FilterFetchingGuard->Val() << ";" + << "AbortsGuard:" << AbortsGuard->Val() << ";"; } TCounterGuard GetResultsForReplyGuard() const { @@ -440,10 +442,14 @@ class TConcreteScanCounters: public TScanCounters { return TCounterGuard(FilterFetchingGuard); } + TCounterGuard GetAbortsGuard() const { + return TCounterGuard(AbortsGuard); + } + bool InWaiting() const { return MergeTasksCount->Val() || AssembleTasksCount->Val() || ReadTasksCount->Val() || ResourcesAllocationTasksCount->Val() || FetchAccessorsCount->Val() || ResultsForSourceCount->Val() || FetchBlobsCount->Val() || ResultsForReplyGuard->Val() || - FilterFetchingGuard->Val(); + FilterFetchingGuard->Val() || AbortsGuard->Val(); } const THashMap>& GetSkipStats() const { diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h index a40d915e04fb..b8b0c19f782c 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h @@ -90,7 +90,7 @@ class TReadContext { void AbortWithError(const TString& errorMessage) { if (AbortionFlag->Inc() == 1) { NActors::TActivationContext::Send(ScanActorId, std::make_unique( - TConclusionStatus::Fail(errorMessage), Counters.GetResultsForReplyGuard())); + TConclusionStatus::Fail(errorMessage), Counters.GetAbortsGuard())); } } diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index f6de383c6548..bce7f1bb383f 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -83,9 +83,8 @@ void TColumnShardScan::HandleScan(NColumnShard::TEvPrivate::TEvTaskProcessedResu Finish(NColumnShard::TScanCounters::EStatusFinish::ConveyorInternalError); } else { ACFL_DEBUG("event", "TEvTaskProcessedResult"); - auto t = static_pointer_cast(result.GetResult()); if (!ScanIterator->Finished()) { - ScanIterator->Apply(t); + ScanIterator->Apply(result.GetResult()); } } ContinueProcessing(); From db504fc2792a97fd6b443197a44ff834f53583e6 Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Tue, 10 Jun 2025 18:24:51 +0300 Subject: [PATCH 11/62] Experimental tiling compaction (#19499) --- ydb/core/protos/flat_scheme_op.proto | 5 + .../storage/optimizer/tiling/tiling.cpp | 709 ++++++++++++++++++ .../engines/storage/optimizer/tiling/ya.make | 15 + .../engines/storage/optimizer/ya.make | 1 + 4 files changed, 730 insertions(+) create mode 100644 ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/optimizer/tiling/ya.make diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 0442e925913a..041c14d2edbb 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -586,10 +586,15 @@ message TCompactionPlannerConstructorContainer { repeated TCompactionSelectorConstructorContainer Selectors = 2; } + message TTilingOptimizer { + optional string Json = 1; + } + oneof Implementation { TLOptimizer LBuckets = 20; TSOptimizer SBuckets = 21; TLCOptimizer LCBuckets = 22; + TTilingOptimizer Tiling = 23; } } diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp new file mode 100644 index 000000000000..bea3b1b159fe --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp @@ -0,0 +1,709 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace NKikimr::NOlap::NStorageOptimizer::NTiling { + +struct TSettings { + unsigned Factor = 10; + unsigned MaxLevels = 10; + unsigned ExpectedPortionCount = 1; + ui64 ExpectedPortionSize = 4 * 1024 * 1024; // 4MiB + unsigned MaxAccumulateCount = 10; + ui64 MaxAccumulatePortionSize = 512 * 1024; // 512KiB + TDuration MaxAccumulateTime = TDuration::Seconds(60); + ui64 MaxCompactionBytes = 256 * 1024 * 1024; + ui32 FullCompactionUntilLevel = 0; + ui64 FullCompactionMaxBytes = -1; + bool CompactNextLevelEdges = false; + + NJson::TJsonValue SettingsJson; + + using TJsonValueHandler = std::function; + + template + static std::pair MakeBooleanHandler(TStringBuf name, TCallback&& callback) { + return { + name, + [name, callback](TSettings& self, const NJson::TJsonValue& value) { + if (!value.IsBoolean()) { + return TConclusionStatus::Fail(TStringBuilder() << name << " must be a boolean"); + } + callback(self, value.GetBoolean()); + return TConclusionStatus::Success(); + } + }; + } + + template + static std::pair MakeNumberHandler(TStringBuf name, TCallback&& callback) { + return { + name, + [name, callback](TSettings& self, const NJson::TJsonValue& value) { + if (!value.IsUInteger()) { + return TConclusionStatus::Fail(TStringBuilder() << name << " must be a number"); + } + callback(self, value.GetUInteger()); + return TConclusionStatus::Success(); + } + }; + } + + template + static std::pair MakeDurationHandler(TStringBuf name, TCallback&& callback) { + return { + name, + [name, callback](TSettings& self, const NJson::TJsonValue& value) { + if (!value.IsString()) { + return TConclusionStatus::Fail(TStringBuilder() << name << " must be a duration string"); + } + TDuration d; + if (!TDuration::TryParse(value.GetString(), d)) { + return TConclusionStatus::Fail(TStringBuilder() << name << " value cannot be parsed as duration: " << value.GetString()); + } + callback(self, d); + return TConclusionStatus::Success(); + } + }; + } + + static inline THashMap JsonValueHandlers = { + MakeNumberHandler("factor", [](auto& self, auto value) { self.Factor = value; }), + MakeNumberHandler("max_levels", [](auto& self, auto value) { self.MaxLevels = value; }), + MakeNumberHandler("expected_portion_count", [](auto& self, auto value) { self.ExpectedPortionCount = value; }), + MakeNumberHandler("expected_portion_size", [](auto& self, auto value) { self.ExpectedPortionSize = value; }), + MakeNumberHandler("max_accumulate_count", [](auto& self, auto value) { self.MaxAccumulateCount = value; }), + MakeNumberHandler("max_accumulate_portion_size", [](auto& self, auto value) { self.MaxAccumulatePortionSize = value; }), + MakeDurationHandler("max_accumulate_time", [](auto& self, auto value) { self.MaxAccumulateTime = value; }), + MakeNumberHandler("max_compaction_bytes", [](auto& self, auto value) { self.MaxCompactionBytes = value; }), + MakeNumberHandler("full_compaction_until_level", [](auto& self, auto value) { self.FullCompactionUntilLevel = value; }), + MakeNumberHandler("full_compaction_max_bytes", [](auto& self, auto value) { self.FullCompactionMaxBytes = value; }), + MakeBooleanHandler("compact_next_level_edges", [](auto& self, auto value) { self.CompactNextLevelEdges = value; }), + }; + + void SerializeToProto(NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TTilingOptimizer& proto) const { + if (SettingsJson.IsDefined()) { + proto.SetJson(NJson::WriteJson(SettingsJson)); + } + } + + TConclusionStatus DeserializeFromProto(const NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TTilingOptimizer& proto) { + if (proto.HasJson()) { + NJson::TJsonValue jsonInfo; + if (!NJson::ReadJsonFastTree(proto.GetJson(), &jsonInfo)) { + return TConclusionStatus::Fail("cannot parse previously serialized json"); + } + return DeserializeFromJson(jsonInfo); + } + return TConclusionStatus::Success(); + } + + TConclusionStatus DeserializeFromJson(const NJson::TJsonValue& jsonInfo) { + if (!jsonInfo.IsMap()) { + return TConclusionStatus::Fail("settings must be an object"); + } + + for (const auto& [name, value] : jsonInfo.GetMapSafe()) { + auto it = JsonValueHandlers.find(name); + if (it == JsonValueHandlers.end()) { + return TConclusionStatus::Fail(TStringBuilder() << "unknown tiling compaction setting " << name); + } + TConclusionStatus status = it->second(*this, value); + if (!status.IsSuccess()) { + return status; + } + } + + SettingsJson = jsonInfo; + + return TConclusionStatus::Success(); + } +}; + +using TPortionMap = THashMap; + +struct TAccumulator { + const ui32 Level; + TPortionMap Portions; + ui64 TotalBlobBytes = 0; + + TPortionMap Compacting; + + TInstant LastCompaction = {}; + bool TimeExceeded = false; + + explicit TAccumulator(ui32 level) + : Level(level) + {} + + bool Empty() const { + return Portions.empty() && Compacting.empty(); + } + + void OnSettingsChanged(const TSettings&) { + TimeExceeded = false; + } + + void Actualize(const TSettings& settings, TInstant now) { + if (Level == 0 && LastCompaction && (now - LastCompaction) >= settings.MaxAccumulateTime) { + TimeExceeded = true; + } + } + + bool NeedCompaction(const TSettings& settings) const { + if (Portions.size() < 2) { + return false; + } + return ( + TimeExceeded || + Portions.size() >= settings.MaxAccumulateCount || + TotalBlobBytes >= settings.ExpectedPortionCount * settings.ExpectedPortionSize); + } + + TPortionInfo::TConstPtr RemoveForCompaction(TPortionMap::iterator it) { + auto p = it->second; + Compacting[p->GetPortionId()] = p; + TotalBlobBytes -= p->GetTotalBlobBytes(); + Portions.erase(it); + return p; + } + + std::vector StartCompaction(const TSettings& settings, const std::shared_ptr& locksManager) { + std::vector portions; + ui64 totalBytes = 0; + auto it = Portions.begin(); + while (it != Portions.end()) { + if (locksManager->IsLocked(*it->second, NDataLocks::ELockCategory::Compaction)) { + // Skip locked portions + ++it; + continue; + } + auto p = RemoveForCompaction(it++); + totalBytes += p->GetTotalBlobBytes(); + portions.push_back(std::move(p)); + if (portions.size() >= 2 && totalBytes >= settings.MaxCompactionBytes) { + break; + } + } + if (!portions.empty()) { + LastCompaction = TInstant::Now(); + TimeExceeded = false; + } + return portions; + } + + void Add(const TPortionInfo::TPtr& p) { + if (Portions.empty()) { + LastCompaction = TInstant::Now(); + } + Remove(p->GetPortionId()); + Portions[p->GetPortionId()] = p; + TotalBlobBytes += p->GetTotalBlobBytes(); + TimeExceeded = false; + LastCompaction = TInstant::Now(); + } + + void Remove(ui64 id) { + Compacting.erase(id); + auto it = Portions.find(id); + if (it != Portions.end()) { + TotalBlobBytes -= it->second->GetTotalBlobBytes(); + Portions.erase(it); + } + } +}; + +struct TSimpleKeyCompare { + std::partial_ordering operator()(const NArrow::TSimpleRow& a, const NArrow::TSimpleRow& b) const { + return a.CompareNotNull(b); + } +}; + +struct TLevel { + const ui32 Level; + TPortionMap Portions; + TIntersectionTree Intersections; + ui64 TotalBlobBytes = 0; + + TPortionMap Compacting; + bool CheckCompactions = true; + + TLevel* Next = nullptr; + + TLevel(ui32 level) + : Level(level) + {} + + bool Empty() const { + return Portions.empty() && Compacting.empty(); + } + + void OnSettingsChanged(const TSettings&) { + CheckCompactions = true; + } + + bool NeedCompaction(const TSettings& settings) const { + if (Portions.size() < 2) { + return false; + } + return Intersections.GetMaxCount() >= i32(settings.Factor); + } + + void Add(const TPortionInfo::TPtr& p) { + Remove(p->GetPortionId()); + Portions[p->GetPortionId()] = p; + TotalBlobBytes += p->GetTotalBlobBytes(); + Intersections.Add(p->GetPortionId(), p->IndexKeyStart(), p->IndexKeyEnd()); + CheckCompactions = true; + } + + void Remove(ui64 id) { + if (Compacting.erase(id)) { + CheckCompactions = true; + } + + auto it = Portions.find(id); + if (it != Portions.end()) { + const auto& p = it->second; + Intersections.Remove(p->GetPortionId()); + TotalBlobBytes -= p->GetTotalBlobBytes(); + Portions.erase(it); + CheckCompactions = true; + } + } + + TPortionInfo::TPtr FindPortion(ui64 id) { + auto it = Portions.find(id); + if (it != Portions.end()) { + return it->second; + } + return nullptr; + } + + TPortionInfo::TConstPtr RemoveForCompaction(TPortionMap::iterator it) { + auto p = it->second; + Intersections.Remove(p->GetPortionId()); + TotalBlobBytes -= p->GetTotalBlobBytes(); + Compacting[p->GetPortionId()] = p; + Portions.erase(it); + return p; + } + + TPortionInfo::TConstPtr RemoveForCompaction(ui64 id) { + auto it = Portions.find(id); + if (it != Portions.end()) { + return RemoveForCompaction(it); + } + return nullptr; + } + + std::vector StartCompaction(const TSettings& settings, const std::shared_ptr& locksManager) { + std::vector portions; + if (auto range = Intersections.GetMaxRange()) { + bool fullCompaction = Level < settings.FullCompactionUntilLevel && TotalBlobBytes <= Min(settings.MaxCompactionBytes, settings.FullCompactionMaxBytes); + bool compactNextLevelEdges = Next && settings.CompactNextLevelEdges && !fullCompaction; + + std::vector candidates; + if (fullCompaction) { + for (const auto& [id, p] : Portions) { + candidates.push_back(id); + } + } else { + range.ForEachValue([&](ui64 id) { + candidates.push_back(id); + return true; + }); + } + + std::vector selected; + selected.reserve(candidates.size()); + ui64 selectedBytes = 0; + for (ui64 id : candidates) { + auto it = Portions.find(id); + if (it == Portions.end()) { + continue; // shouldn't happen + } + const auto& p = it->second; + if (locksManager->IsLocked(*p, NDataLocks::ELockCategory::Compaction)) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: skipping level (portions locked)")("level", Level); + CheckCompactions = false; + return {}; + } + selected.push_back(id); + selectedBytes += p->GetTotalBlobBytes(); + if (selected.size() >= 2 && selectedBytes >= settings.MaxCompactionBytes) { + break; + } + } + + std::optional minKey; + std::optional maxKey; + portions.reserve(selected.size()); + for (ui64 id : selected) { + auto it = Portions.find(id); + AFL_VERIFY(it != Portions.end()); + auto p = RemoveForCompaction(it); + portions.push_back(p); + if (compactNextLevelEdges) { + if (auto key = p->IndexKeyStart(); !minKey || key < *minKey) { + minKey = key; + } + if (auto key = p->IndexKeyEnd(); !maxKey || key > *maxKey) { + maxKey = key; + } + } + } + + if (compactNextLevelEdges && minKey && maxKey) { + TPortionInfo::TPtr bestLeft, bestRight; + if (auto minFound = Next->Intersections.FindRange(*minKey); minFound && minFound.GetCount() > 0) { + minFound.ForEachValue([&](ui64 id) { + if (auto p = Next->FindPortion(id); p && !locksManager->IsLocked(*p, NDataLocks::ELockCategory::Compaction)) { + // The best portion doesn't extend our compaction range by much + if (p->IndexKeyStart() < *minKey && p->IndexKeyEnd() < *maxKey && + (!bestLeft || bestLeft->IndexKeyStart() < p->IndexKeyStart())) { + bestLeft = p; + } + } + return true; + }); + } + if (auto maxFound = Next->Intersections.FindRange(*maxKey); maxFound && maxFound.GetCount() > 0) { + maxFound.ForEachValue([&](ui64 id) { + if (auto p = Next->FindPortion(id); p && !locksManager->IsLocked(*p, NDataLocks::ELockCategory::Compaction)) { + // The best portion doesn't extend our compaction range by much + if (*minKey < p->IndexKeyStart() && *maxKey < p->IndexKeyEnd() && + (!bestRight || bestRight->IndexKeyEnd() > p->IndexKeyEnd())) { + bestRight = p; + } + } + return true; + }); + } + if (bestLeft) { + if (auto p = Next->RemoveForCompaction(bestLeft->GetPortionId())) { + portions.push_back(p); + } + } + if (bestRight && bestRight != bestLeft) { + if (auto p = Next->RemoveForCompaction(bestRight->GetPortionId())) { + portions.push_back(p); + } + } + } + } + return portions; + } +}; + +class TOptimizerPlanner : public IOptimizerPlanner, private TSettings { + using TBase = IOptimizerPlanner; + +public: + TOptimizerPlanner(const TInternalPathId pathId, const std::shared_ptr& storagesManager, + const std::shared_ptr& primaryKeysSchema, const TSettings& settings = {}) + : TBase(pathId) + , TSettings(settings) + , StoragesManager(storagesManager) + , PrimaryKeysSchema(primaryKeysSchema) + { + } + + bool UpdateSettings(const TSettings& settings) { + if (settings.MaxLevels != MaxLevels || settings.MaxAccumulatePortionSize != MaxAccumulatePortionSize) { + // These settings affect portion distribution, cannot apply + return false; + } + // Update settings + static_cast(*this) = settings; + for (auto& acc : Accumulator) { + acc.OnSettingsChanged(settings); + } + for (auto& level : Levels) { + level.OnSettingsChanged(settings); + } + return true; + } + +private: + NArrow::NMerger::TIntervalPositions GetBucketPositions() const override { + // We allow and encourage intersections + return {}; + } + + std::vector DoGetTasksDescription() const override { + // TODO + return {}; + } + + bool DoIsLocked(const std::shared_ptr& dataLocksManager) const override { + // This method is never used + Y_UNUSED(dataLocksManager); + return false; + } + + bool IsAccumulatorPortion(const TPortionInfo::TPtr& p) { + if (p->GetTotalBlobBytes() <= MaxAccumulatePortionSize) { + return true; // portion is too small + } + return false; + } + + void DoModifyPortions(const THashMap& add, const THashMap& remove) override { + std::vector sortedRemove; + for (const auto& [_, p] : remove) { + sortedRemove.push_back(p); + } + std::sort(sortedRemove.begin(), sortedRemove.end(), [](const auto& a, const auto& b) { + return b->GetPortionId() < a->GetPortionId(); + }); + + for (const auto& p : sortedRemove) { + ui32 level = p->GetCompactionLevel(); + if (level < Accumulator.size()) { + Accumulator[level].Remove(p->GetPortionId()); + } + if (level < Levels.size()) { + Levels[level].Remove(p->GetPortionId()); + } + } + + std::vector sortedAdd; + for (const auto& [_, p] : add) { + sortedAdd.push_back(p); + } + std::sort(sortedAdd.begin(), sortedAdd.end(), [](const auto& a, const auto& b) { + return a->GetPortionId() < b->GetPortionId(); + }); + + for (const auto& p : sortedAdd) { + switch (p->GetProduced()) { + case NPortion::INACTIVE: + case NPortion::EVICTED: + break; + default: { + ui32 level = p->GetCompactionLevel(); + if (level >= MaxLevels) { + level = MaxLevels - 1; + p->MutableMeta().ResetCompactionLevel(level); + } + if (IsAccumulatorPortion(p)) { + EnsureAccumulator(level).Add(p); + } else { + EnsureLevel(level).Add(p); + } + break; + } + } + } + } + + TLevel& EnsureLevel(ui32 level) { + while (level >= Levels.size()) { + ui32 next = Levels.size(); + Levels.emplace_back(next); + if (next > 0) { + Levels[next - 1].Next = &Levels.back(); + } + } + return Levels[level]; + } + + bool NeedLevelCompaction(ui32 level) const { + return Levels[level].NeedCompaction(*this); + } + + std::shared_ptr GetCompactLevelTask( + const std::shared_ptr& granule, + const std::shared_ptr& locksManager, + ui32 level) const { + if (!NeedLevelCompaction(level)) { + return nullptr; + } + + std::vector portions = Levels[level].StartCompaction(*this, locksManager); + if (portions.empty()) { + return nullptr; + } + + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: compacting level")("level", level)("count", portions.size()); + + ui32 targetLevel = level + 1; + if (targetLevel >= MaxLevels) { + targetLevel = MaxLevels - 1; + } + + TSaverContext saverContext(StoragesManager); + auto result = std::make_shared(granule, portions, saverContext); + result->SetTargetCompactionLevel(targetLevel); + result->SetPortionExpectedSize(ExpectedPortionSize); + return result; + } + + TAccumulator& EnsureAccumulator(ui32 level) { + while (level >= Accumulator.size()) { + ui32 next = Accumulator.size(); + Accumulator.emplace_back(next); + } + return Accumulator[level]; + } + + bool NeedAccumulatorCompaction(ui32 level) const { + return Accumulator[level].NeedCompaction(*this); + } + + std::shared_ptr GetCompactAccumulatorTask( + const std::shared_ptr& granule, + const std::shared_ptr& locksManager, + ui32 level) const { + if (!NeedAccumulatorCompaction(level)) { + return nullptr; + } + + std::vector portions = Accumulator[level].StartCompaction(*this, locksManager); + if (portions.empty()) { + return nullptr; + } + + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: compacting accumulator")("level", level)("count", portions.size()); + + ui32 targetLevel = level + 1; + if (targetLevel >= MaxLevels) { + targetLevel = MaxLevels - 1; + } + + TSaverContext saverContext(StoragesManager); + auto result = std::make_shared(granule, portions, saverContext); + result->SetTargetCompactionLevel(targetLevel); + result->SetPortionExpectedSize(ExpectedPortionSize); + return result; + } + + std::shared_ptr DoGetOptimizationTask(std::shared_ptr granule, const std::shared_ptr& locksManager) const override { + // Check compactions, top to bottom + for (size_t level = 0; level < Max(Accumulator.size(), Levels.size()); ++level) { + if (level < Accumulator.size()) { + if (auto result = GetCompactAccumulatorTask(granule, locksManager, level)) { + return result; + } + } + if (level < Levels.size()) { + if (auto result = GetCompactLevelTask(granule, locksManager, level)) { + return result; + } + } + } + + // Nothing to compact + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: nothing to compact"); + return nullptr; + } + + void DoActualize(const TInstant currentInstant) override { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: actualize called"); + for (size_t level = 0; level < Accumulator.size(); ++level) { + Accumulator[level].Actualize(*this, currentInstant); + } + } + + TOptimizationPriority DoGetUsefulMetric() const override { + for (size_t level = 0; level < Max(Accumulator.size(), Levels.size()); ++level) { + if (level < Accumulator.size() && NeedAccumulatorCompaction(level)) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: returning DoGetUsefulMetric for accumulator")("level", level); + return TOptimizationPriority::Critical(Accumulator[level].Portions.size()); + } + if (level < Levels.size() && NeedLevelCompaction(level)) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: returning DoGetUsefulMetric for level")("level", level); + return TOptimizationPriority::Critical(Levels[level].Portions.size()); + } + } + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "tiling compaction: returning DoGetUsefulMetric zero"); + return TOptimizationPriority::Zero(); + } + + TString DoDebugString() const override { + return "TODO: DebugString"; + } + + NJson::TJsonValue DoSerializeToJsonVisual() const override { + return NJson::JSON_NULL; + } + +private: + std::shared_ptr StoragesManager; + std::shared_ptr PrimaryKeysSchema; + + // Accumulator accumulates small portions by size-tiered compaction + mutable std::deque Accumulator; + + // Levels compaction, each level has portions without intersections + mutable std::deque Levels; +}; + +class TOptimizerPlannerConstructor : public IOptimizerPlannerConstructor { +public: + static TString GetClassNameStatic() { + return "tiling"; + } + + TString GetClassName() const override { + return GetClassNameStatic(); + } + +private: + static inline const TFactory::TRegistrator Registrator = + TFactory::TRegistrator(GetClassNameStatic()); + + void DoSerializeToProto(TProto& proto) const override { + Settings.SerializeToProto(*proto.MutableTiling()); + } + + bool DoDeserializeFromProto(const TProto& proto) override { + if (!proto.HasTiling()) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse tiling compaction optimizer from proto")("proto", proto.DebugString()); + return false; + } + auto status = Settings.DeserializeFromProto(proto.GetTiling()); + if (!status.IsSuccess()) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse tiling compaction optimizer from proto")("description", status.GetErrorDescription()); + return false; + } + return true; + } + + TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonInfo) override { + return Settings.DeserializeFromJson(jsonInfo); + } + + bool DoApplyToCurrentObject(IOptimizerPlanner& current) const override { + auto* itemClass = dynamic_cast(¤t); + if (!itemClass) { + return false; + } + return itemClass->UpdateSettings(Settings); + } + + TConclusion> DoBuildPlanner(const TBuildContext& context) const override { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("message", "creating tiling compaction optimizer"); + return std::make_shared(context.GetPathId(), context.GetStorages(), context.GetPKSchema(), Settings); + } + +private: + TSettings Settings; +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer::NTiling diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/ya.make new file mode 100644 index 000000000000..1deebb7b99af --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +SRCS( + GLOBAL tiling.cpp +) + +PEERDIR( + contrib/libs/apache/arrow + ydb/core/protos + ydb/core/formats/arrow + ydb/core/tx/columnshard/engines/changes/abstract + ydb/library/intersection_tree +) + +END() diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make index 2213cc6949c6..310b76bce108 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/ya.make @@ -4,6 +4,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/storage/optimizer/abstract ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets + ydb/core/tx/columnshard/engines/storage/optimizer/tiling ) END() From 24d1e43f2034be8bcafaee35ea30d91edb518c63 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 10 Jun 2025 23:07:05 +0300 Subject: [PATCH 12/62] fix deletion usage on compaction (#19576) --- .../engines/changes/general_compaction.cpp | 23 ++++++++---- .../engines/changes/merge_subset.cpp | 28 +++++++++------ .../engines/changes/merge_subset.h | 36 ++++++++++++------- .../tx/columnshard/hooks/testing/controller.h | 4 --- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 1 - 5 files changed, 57 insertions(+), 35 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 665928fa0b79..f57749aeda42 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -69,9 +69,19 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc AFL_VERIFY(usedPortionIds.emplace(i.GetPortionInfo().GetPortionId()).second); currentToMerge.emplace_back(std::make_shared(std::move(i), GranuleMeta)); } + + const auto buildPortionsToMerge = [&](const std::vector>& toMerge, const bool useDeletion) { + std::vector result; + for (auto&& i : toMerge) { + auto mergePortions = i->BuildPortionsToMerge(context, seqDataColumnIds, resultFiltered, usedPortionIds, useDeletion); + result.insert(result.end(), mergePortions.begin(), mergePortions.end()); + } + return result; + }; + auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); while (true) { - std::vector toMerge; + std::vector> toMerge; ui64 sumMemory = 0; ui64 totalSumMemory = 0; std::vector> appendedToMerge; @@ -80,9 +90,9 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc if (NYDBTest::TControllers::GetColumnShardController()->CheckPortionsToMergeOnCompaction( sumMemory + i->GetColumnMaxChunkMemory(), subsetsCount) && subsetsCount > 1) { - auto merged = BuildAppendedPortionsByChunks(context, std::move(toMerge), resultFiltered, stats); + auto merged = BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, false), resultFiltered, stats); if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared(std::move(merged))); + appendedToMerge.emplace_back(std::make_shared(std::move(merged), GranuleMeta)); } toMerge.clear(); sumMemory = 0; @@ -90,15 +100,14 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc } sumMemory += i->GetColumnMaxChunkMemory(); totalSumMemory += i->GetColumnMaxChunkMemory(); - auto mergePortions = i->BuildPortionsToMerge(context, seqDataColumnIds, resultFiltered, usedPortionIds); - toMerge.insert(toMerge.end(), mergePortions.begin(), mergePortions.end()); + toMerge.emplace_back(i); ++subsetsCount; } if (toMerge.size()) { - auto merged = BuildAppendedPortionsByChunks(context, std::move(toMerge), resultFiltered, stats); + auto merged = BuildAppendedPortionsByChunks(context, buildPortionsToMerge(toMerge, appendedToMerge.empty()), resultFiltered, stats); if (appendedToMerge.size()) { if (merged.size()) { - appendedToMerge.emplace_back(std::make_shared(std::move(merged))); + appendedToMerge.emplace_back(std::make_shared(std::move(merged), GranuleMeta)); } } else { context.Counters.OnCompactionCorrectMemory(totalSumMemory); diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp index 6286026d6a2b..c978142a3465 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.cpp @@ -2,8 +2,9 @@ namespace NKikimr::NOlap::NCompaction { -std::shared_ptr TReadPortionToMerge::BuildPortionFilter(const std::optional& shardingActual, - const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage) const { +std::shared_ptr ISubsetToMerge::BuildPortionFilter(const std::optional& shardingActual, + const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, + const bool useDeletionFilter) const { std::shared_ptr filter; if (shardingActual && pInfo.NeedShardingFilter(*shardingActual)) { std::set fieldNames; @@ -15,7 +16,7 @@ std::shared_ptr TReadPortionToMerge::BuildPortionFilter(c filter = shardingActual->GetShardingInfo()->GetFilter(table); } NArrow::TColumnFilter filterDeleted = NArrow::TColumnFilter::BuildAllowFilter(); - if (pInfo.GetMeta().GetDeletionsCount()) { + if (pInfo.GetMeta().GetDeletionsCount() && useDeletionFilter) { if (pInfo.GetPortionType() == EPortionType::Written) { AFL_VERIFY(pInfo.GetMeta().GetDeletionsCount() == pInfo.GetRecordsCount()); filterDeleted = NArrow::TColumnFilter::BuildDenyFilter(); @@ -45,23 +46,26 @@ std::shared_ptr TReadPortionToMerge::BuildPortionFilter(c } std::vector TReadPortionToMerge::DoBuildPortionsToMerge(const TConstructionContext& context, - const std::set& seqDataColumnIds, const std::shared_ptr& resultFiltered, - const THashSet& usedPortionIds) const { + const std::set& seqDataColumnIds, const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const { auto blobsSchema = ReadPortion.GetPortionInfo().GetSchema(context.SchemaVersions); auto batch = ReadPortion.RestoreBatch(*blobsSchema, *resultFiltered, seqDataColumnIds, false).DetachResult(); auto shardingActual = context.SchemaVersions.GetShardingInfoActual(GranuleMeta->GetPathId()); - std::shared_ptr filter = BuildPortionFilter(shardingActual, batch, ReadPortion.GetPortionInfo(), usedPortionIds); + std::shared_ptr filter = + BuildPortionFilter(shardingActual, batch, ReadPortion.GetPortionInfo(), usedPortionIds, useDeletionFilter); return { TPortionToMerge(batch, filter) }; } std::vector TWritePortionsToMerge::DoBuildPortionsToMerge(const TConstructionContext& context, - const std::set& seqDataColumnIds, const std::shared_ptr& resultFiltered, - const THashSet& /*usedPortionIds*/) const { + const std::set& seqDataColumnIds, const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const { std::vector result; for (auto&& i : WritePortions) { auto blobsSchema = i.GetPortionResult().GetPortionInfo().GetSchema(context.SchemaVersions); auto batch = i.RestoreBatch(*blobsSchema, *resultFiltered, seqDataColumnIds, false).DetachResult(); - result.emplace_back(TPortionToMerge(batch, nullptr)); + std::shared_ptr filter = + BuildPortionFilter(std::nullopt, batch, i.GetPortionResult().GetPortionInfo(), usedPortionIds, useDeletionFilter); + result.emplace_back(TPortionToMerge(batch, filter)); } return result; } @@ -76,8 +80,10 @@ ui64 TWritePortionsToMerge::GetColumnMaxChunkMemory() const { return result; } -TWritePortionsToMerge::TWritePortionsToMerge(std::vector&& portions) - : WritePortions(std::move(portions)) { +TWritePortionsToMerge::TWritePortionsToMerge( + std::vector&& portions, const std::shared_ptr& granuleMeta) + : TBase(granuleMeta) + , WritePortions(std::move(portions)) { ui32 idx = 0; for (auto&& i : WritePortions) { i.GetPortionConstructor().MutablePortionConstructor().SetPortionId(++idx); diff --git a/ydb/core/tx/columnshard/engines/changes/merge_subset.h b/ydb/core/tx/columnshard/engines/changes/merge_subset.h index da4ce7545dd2..03378a59f896 100644 --- a/ydb/core/tx/columnshard/engines/changes/merge_subset.h +++ b/ydb/core/tx/columnshard/engines/changes/merge_subset.h @@ -24,32 +24,42 @@ class TPortionToMerge { class ISubsetToMerge { private: virtual std::vector DoBuildPortionsToMerge(const TConstructionContext& context, const std::set& seqDataColumnIds, - const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds) const = 0; + const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const = 0; + +protected: + const std::shared_ptr GranuleMeta; + std::shared_ptr BuildPortionFilter(const std::optional& shardingActual, + const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage, + const bool useDeletionFilter) const; public: + ISubsetToMerge(const std::shared_ptr& granule) + : GranuleMeta(granule) { + AFL_VERIFY(GranuleMeta); + } virtual ~ISubsetToMerge() = default; std::vector BuildPortionsToMerge(const TConstructionContext& context, const std::set& seqDataColumnIds, - const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds) const { - return DoBuildPortionsToMerge(context, seqDataColumnIds, resultFiltered, usedPortionIds); + const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const { + return DoBuildPortionsToMerge(context, seqDataColumnIds, resultFiltered, usedPortionIds, useDeletionFilter); } virtual ui64 GetColumnMaxChunkMemory() const = 0; }; class TReadPortionToMerge: public ISubsetToMerge { private: + using TBase = ISubsetToMerge; TReadPortionInfoWithBlobs ReadPortion; - const std::shared_ptr GranuleMeta; - - std::shared_ptr BuildPortionFilter(const std::optional& shardingActual, - const std::shared_ptr& batch, const TPortionInfo& pInfo, const THashSet& portionsInUsage) const; virtual std::vector DoBuildPortionsToMerge(const TConstructionContext& context, const std::set& seqDataColumnIds, - const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds) const override; + const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const override; public: TReadPortionToMerge(TReadPortionInfoWithBlobs&& rPortion, const std::shared_ptr& granuleMeta) - : ReadPortion(std::move(rPortion)) - , GranuleMeta(granuleMeta) { + : TBase(granuleMeta) + , ReadPortion(std::move(rPortion)) { } virtual ui64 GetColumnMaxChunkMemory() const override { @@ -63,15 +73,17 @@ class TReadPortionToMerge: public ISubsetToMerge { class TWritePortionsToMerge: public ISubsetToMerge { private: + using TBase = ISubsetToMerge; std::vector WritePortions; virtual std::vector DoBuildPortionsToMerge(const TConstructionContext& context, const std::set& seqDataColumnIds, - const std::shared_ptr& resultFiltered, const THashSet& /*usedPortionIds*/) const override; + const std::shared_ptr& resultFiltered, const THashSet& usedPortionIds, + const bool useDeletionFilter) const override; virtual ui64 GetColumnMaxChunkMemory() const override; public: - TWritePortionsToMerge(std::vector&& portions); + TWritePortionsToMerge(std::vector&& portions, const std::shared_ptr& granuleMeta); }; } // namespace NKikimr::NOlap::NCompaction diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index 91f7f81943e6..e389c9c4c63a 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -27,7 +27,6 @@ class TController: public TReadOnlyController { YDB_ACCESSOR_DEF(std::optional, OverrideCompactionActualizationLag); YDB_ACCESSOR_DEF(std::optional, OverrideTasksActualizationLag); YDB_ACCESSOR_DEF(std::optional, OverrideMaxReadStaleness); - YDB_ACCESSOR_DEF(std::optional, OverrideAllowMergeFull); YDB_ACCESSOR(std::optional, OverrideMemoryLimitForPortionReading, 100); YDB_ACCESSOR(std::optional, OverrideLimitForPortionsMetadataAsk, 1); YDB_ACCESSOR(std::optional, OverrideBlobSplitSettings, NOlap::NSplitter::TSplitSettings::BuildForTests()); @@ -242,9 +241,6 @@ class TController: public TReadOnlyController { public: virtual bool CheckPortionsToMergeOnCompaction(const ui64 /*memoryAfterAdd*/, const ui32 currentSubsetsCount) override { - if (OverrideAllowMergeFull && *OverrideAllowMergeFull) { - return false; - } return currentSubsetsCount > 1; } diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 678699e4683c..c39db923f860 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -1023,7 +1023,6 @@ Y_UNIT_TEST_SUITE(TOlap) { csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideMaxReadStaleness(TDuration::Seconds(1)); - csController->SetOverrideAllowMergeFull(true); // disable stats batching auto& appData = runtime.GetAppData(); From f1f4a4e9468e87d255020b3b0e59ccbfd1390831 Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Wed, 11 Jun 2025 17:06:47 +0300 Subject: [PATCH 13/62] Fix optimizer comparison when protobuf serializations are the same (#19596) --- .../columnshard/engines/storage/optimizer/abstract/optimizer.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h index 5d783ad785ee..458a91012080 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h @@ -244,6 +244,9 @@ class IOptimizerPlannerConstructor { bool IsEqualTo(const std::shared_ptr& item) const { AFL_VERIFY(!!item); + if (GetClassName() != item->GetClassName()) { + return false; + } TProto selfProto; TProto itemProto; SerializeToProto(selfProto); From cd18c9e8554c157d5d5f735f25c273151b69a4f8 Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Wed, 11 Jun 2025 18:43:22 +0300 Subject: [PATCH 14/62] Add basic validation for tiling compaction settings (#19599) --- .../storage/optimizer/tiling/tiling.cpp | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp index bea3b1b159fe..d8f1e38e3628 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp @@ -128,6 +128,34 @@ struct TSettings { } } + if (Factor < 2) { + return TConclusionStatus::Fail("factor must be at least 2"); + } + + if (MaxLevels < 1 || MaxLevels > 10) { + return TConclusionStatus::Fail("max_levels must be between 1 and 10"); + } + + if (MaxAccumulateCount < 2) { + return TConclusionStatus::Fail("max_accumulate_count must be at least 2"); + } + + if (ExpectedPortionSize < MaxAccumulatePortionSize * 3) { + return TConclusionStatus::Fail(TStringBuilder() + << "expected_portion_size (" << ExpectedPortionSize << ") must be at least " + << "3x larger than max_accumulate_portion_size (" << MaxAccumulatePortionSize << ")"); + } + + if (MaxAccumulateTime < TDuration::Seconds(15)) { + return TConclusionStatus::Fail("max_accumulate_time must be at least 15 seconds"); + } + + if (MaxCompactionBytes < ExpectedPortionSize * 4) { + return TConclusionStatus::Fail(TStringBuilder() + << "max_compaction_bytes (" << MaxCompactionBytes << ") is too small " + << "relative to expected_portion_size (" << ExpectedPortionSize << ")"); + } + SettingsJson = jsonInfo; return TConclusionStatus::Success(); From 7b6f16364eca031292d174707714b370ae59aac8 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Wed, 11 Jun 2025 21:59:52 +0300 Subject: [PATCH 15/62] clean deprecated immediate write option (#19597) --- ydb/core/transfer/transfer_writer.cpp | 2 +- .../blobs_action/transaction/tx_write.cpp | 108 ++++++-------- ydb/core/tx/columnshard/columnshard.h | 8 - .../tx/columnshard/columnshard__write.cpp | 137 +----------------- ydb/core/tx/columnshard/columnshard_impl.h | 2 - .../test_helper/columnshard_ut_common.cpp | 21 --- .../test_helper/columnshard_ut_common.h | 4 - .../tx/data_events/columnshard_splitter.cpp | 3 +- .../tx/data_events/columnshard_splitter.h | 21 +-- ydb/core/tx/data_events/shard_writer.cpp | 41 +----- ydb/core/tx/data_events/shard_writer.h | 19 +-- ydb/core/tx/data_events/shards_splitter.h | 1 - ydb/core/tx/tx_proxy/rpc_long_tx.cpp | 26 ++-- .../tx/tx_proxy/upload_rows_common_impl.h | 11 +- 14 files changed, 79 insertions(+), 325 deletions(-) diff --git a/ydb/core/transfer/transfer_writer.cpp b/ydb/core/transfer/transfer_writer.cpp index 6cea16fbb4f3..59a6a0a8397a 100644 --- a/ydb/core/transfer/transfer_writer.cpp +++ b/ydb/core/transfer/transfer_writer.cpp @@ -352,7 +352,7 @@ class TColumnTableState : public ITableKindState { Issues = std::make_shared(); NTxProxy::DoLongTxWriteSameMailbox(TActivationContext::AsActorContext(), SelfId /* replyTo */, { /* longTxId */ }, { /* dedupId */ }, - NavigateResult->DatabaseName, Path, NavigateResult, Data, Issues, true /* noTxWrite */); + NavigateResult->DatabaseName, Path, NavigateResult, Data, Issues); }; if (Data) { diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 451a29e682d7..bdcb27862bbe 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -46,35 +46,20 @@ bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { Y_ABORT_UNLESS(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetTableId(), minReadSnapshot)); txc.DB.NoMoreReadsForTx(); TWriteOperation::TPtr operation; - if (writeMeta.HasLongTxId()) { - NIceDb::TNiceDb db(txc.DB); - const TInsertWriteId insertWriteId = - Self->GetLongTxWrite(db, writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId(), writeMeta.GetGranuleShardingVersion()); - aggr->AddInsertWriteId(insertWriteId); - if (writeMeta.IsGuaranteeWriter()) { - AFL_VERIFY(aggr->GetSplittedBlobs().size() == 1)("count", aggr->GetSplittedBlobs().size()); + AFL_VERIFY(!writeMeta.HasLongTxId()); + operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); + for (auto&& i : aggr->GetSplittedBlobs()) { + if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { + static TAtomicCounter Counter = 0; + const TInsertWriteId insertWriteId = (TInsertWriteId)Counter.Inc(); + AFL_VERIFY(CommitOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( + "size", aggr->GetSplittedBlobs().size()); } else { - AFL_VERIFY(aggr->GetSplittedBlobs().size() <= 1)("count", aggr->GetSplittedBlobs().size()); - } - if (aggr->GetSplittedBlobs().size() == 1) { - AFL_VERIFY(InsertOneBlob(txc, aggr->GetSplittedBlobs().front(), insertWriteId))("write_id", writeMeta.GetWriteId())( - "insert_write_id", insertWriteId); - } - } else { - operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); - for (auto&& i : aggr->GetSplittedBlobs()) { - if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - static TAtomicCounter Counter = 0; - const TInsertWriteId insertWriteId = (TInsertWriteId)Counter.Inc(); - AFL_VERIFY(CommitOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( - "size", aggr->GetSplittedBlobs().size()); - } else { - const TInsertWriteId insertWriteId = Self->InsertTable->BuildNextWriteId(txc); - aggr->AddInsertWriteId(insertWriteId); - AFL_VERIFY(InsertOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( - "size", aggr->GetSplittedBlobs().size()); - } + const TInsertWriteId insertWriteId = Self->InsertTable->BuildNextWriteId(txc); + aggr->AddInsertWriteId(insertWriteId); + AFL_VERIFY(InsertOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( + "size", aggr->GetSplittedBlobs().size()); } } } @@ -90,30 +75,24 @@ bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { Results.clear(); for (auto&& aggr : buffer.GetAggregations()) { const auto& writeMeta = aggr->GetWriteMeta(); - if (!writeMeta.HasLongTxId()) { - auto operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); - operation->OnWriteFinish(txc, aggr->GetInsertWriteIds(), operation->GetBehaviour() == EOperationBehaviour::NoTxWrite); - Self->OperationsManager->LinkInsertWriteIdToOperationWriteId(aggr->GetInsertWriteIds(), operation->GetWriteId()); - if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID()); - Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); - } else { - auto& info = Self->OperationsManager->GetLockVerified(operation->GetLockId()); - NKikimrDataEvents::TLock lock; - lock.SetLockId(operation->GetLockId()); - lock.SetDataShard(Self->TabletID()); - lock.SetGeneration(info.GetGeneration()); - lock.SetCounter(info.GetInternalGenerationCounter()); - lock.SetPathId(writeMeta.GetTableId().GetRawValue()); - auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); - Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); - } + AFL_VERIFY(!writeMeta.HasLongTxId()); + auto operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); + operation->OnWriteFinish(txc, aggr->GetInsertWriteIds(), operation->GetBehaviour() == EOperationBehaviour::NoTxWrite); + Self->OperationsManager->LinkInsertWriteIdToOperationWriteId(aggr->GetInsertWriteIds(), operation->GetWriteId()); + if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { + auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID()); + Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); } else { - Y_ABORT_UNLESS(aggr->GetInsertWriteIds().size() == 1); - auto ev = std::make_unique( - Self->TabletID(), writeMeta, (ui64)aggr->GetInsertWriteIds().front(), NKikimrTxColumnShard::EResultStatus::SUCCESS); - Results.emplace_back(std::move(ev), writeMeta.GetSource(), 0); + auto& info = Self->OperationsManager->GetLockVerified(operation->GetLockId()); + NKikimrDataEvents::TLock lock; + lock.SetLockId(operation->GetLockId()); + lock.SetDataShard(Self->TabletID()); + lock.SetGeneration(info.GetGeneration()); + lock.SetCounter(info.GetInternalGenerationCounter()); + lock.SetPathId(writeMeta.GetTableId().GetRawValue()); + auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); + Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); } } return true; @@ -138,22 +117,21 @@ void TTxWrite::DoComplete(const TActorContext& ctx) { } for (ui32 i = 0; i < buffer.GetAggregations().size(); ++i) { const auto& writeMeta = buffer.GetAggregations()[i]->GetWriteMeta(); - if (!writeMeta.HasLongTxId()) { - auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetTableId())) { - auto evWrite = std::make_shared(writeMeta.GetTableId(), - buffer.GetAggregations()[i]->GetRecordBatch(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); - Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); - } - } - if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - Self->OperationsManager->AddTemporaryTxLink(op->GetLockId()); - AFL_VERIFY(CommitSnapshot); - Self->OperationsManager->CommitTransactionOnComplete(*Self, op->GetLockId(), *CommitSnapshot); - Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED); + AFL_VERIFY(!writeMeta.HasLongTxId()); + auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { + if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetTableId())) { + auto evWrite = std::make_shared(writeMeta.GetTableId(), + buffer.GetAggregations()[i]->GetRecordBatch(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); + Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); } } + if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { + Self->OperationsManager->AddTemporaryTxLink(op->GetLockId()); + AFL_VERIFY(CommitSnapshot); + Self->OperationsManager->CommitTransactionOnComplete(*Self, op->GetLockId(), *CommitSnapshot); + Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED); + } Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant()); Self->Counters.GetCSCounters().OnSuccessWriteResponse(); } diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h index 283324fbb8d9..1e45af970e3e 100644 --- a/ydb/core/tx/columnshard/columnshard.h +++ b/ydb/core/tx/columnshard/columnshard.h @@ -303,14 +303,6 @@ inline auto& Proto(TEvColumnShard::TEvProposeTransactionResult* ev) { return ev->Record; } -inline auto& Proto(TEvColumnShard::TEvWrite* ev) { - return ev->Record; -} - -inline auto& Proto(TEvColumnShard::TEvWriteResult* ev) { - return ev->Record; -} - inline TMessageSeqNo SeqNoFromProto(const NKikimrTxColumnShard::TSchemaSeqNo& proto) { return TMessageSeqNo(proto.GetGeneration(), proto.GetRound()); } diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 1c7960226e2c..12967588a2b6 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -152,24 +152,11 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo Counters.GetCSCounters().OnWritePutBlobsFail(TMonotonic::Now() - writeMeta.GetWriteStartInstant()); Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL); - auto errCode = NKikimrTxColumnShard::EResultStatus::STORAGE_ERROR; - if (putResult.GetPutStatus() == NKikimrProto::TIMEOUT || putResult.GetPutStatus() == NKikimrProto::DEADLINE) { - errCode = NKikimrTxColumnShard::EResultStatus::TIMEOUT; - } else if (putResult.GetPutStatus() == NKikimrProto::TRYLATER || putResult.GetPutStatus() == NKikimrProto::OUT_OF_SPACE) { - errCode = NKikimrTxColumnShard::EResultStatus::OVERLOADED; - } else if (putResult.GetPutStatus() == NKikimrProto::CORRUPTED) { - errCode = NKikimrTxColumnShard::EResultStatus::ERROR; - } - - if (writeMeta.HasLongTxId()) { - auto result = std::make_unique(TabletID(), writeMeta, errCode); - ctx.Send(writeMeta.GetSource(), result.release()); - } else { - auto operation = OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), operation->GetLockId(), - ev->Get()->GetWriteResultStatus(), ev->Get()->GetErrorMessage() ? ev->Get()->GetErrorMessage() : "put data fails"); - ctx.Send(writeMeta.GetSource(), result.release(), 0, operation->GetCookie()); - } + AFL_VERIFY(!writeMeta.HasLongTxId()); + auto operation = OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), operation->GetLockId(), + ev->Get()->GetWriteResultStatus(), ev->Get()->GetErrorMessage() ? ev->Get()->GetErrorMessage() : "put data fails"); + ctx.Send(writeMeta.GetSource(), result.release(), 0, operation->GetCookie()); Counters.GetCSCounters().OnFailedWriteResponse(EWriteFailReason::PutBlob); wBuffer.RemoveData(aggr, StoragesManager->GetInsertOperator()); } else { @@ -187,120 +174,6 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteDraft::TPtr& ev, const TActorConte Execute(new TTxWriteDraft(this, ev->Get()->WriteController), ctx); } -void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContext& ctx) { - Counters.GetCSCounters().OnStartWriteRequest(); - - const auto& record = Proto(ev->Get()); - const auto pathId = TInternalPathId::FromRawValue(record.GetTableId()); - const ui64 writeId = record.GetWriteId(); - const ui64 cookie = ev->Cookie; - const TString dedupId = record.GetDedupId(); - const auto source = ev->Sender; - - Counters.GetColumnTablesCounters()->GetPathIdCounter(pathId)->OnWriteEvent(); - - std::optional granuleShardingVersion; - if (record.HasGranuleShardingVersion()) { - granuleShardingVersion = record.GetGranuleShardingVersion(); - } - - auto writeMetaPtr = std::make_shared(writeId, pathId, source, granuleShardingVersion, - TGUID::CreateTimebased().AsGuidString(), Counters.GetCSCounters().WritingCounters->GetWriteFlowCounters()); - auto& writeMeta = *writeMetaPtr; - if (record.HasModificationType()) { - writeMeta.SetModificationType(TEnumOperator::DeserializeFromProto(record.GetModificationType())); - } - writeMeta.SetDedupId(dedupId); - Y_ABORT_UNLESS(record.HasLongTxId()); - writeMeta.SetLongTxId(NLongTxService::TLongTxId::FromProto(record.GetLongTxId())); - writeMeta.SetWritePartId(record.GetWritePartId()); - - const auto returnFail = [&](const NColumnShard::ECumulativeCounters signalIndex, const EWriteFailReason reason, - NKikimrTxColumnShard::EResultStatus resultStatus) { - Counters.GetTabletCounters()->IncCounter(signalIndex); - - ctx.Send(source, std::make_unique(TabletID(), writeMeta, resultStatus)); - Counters.GetCSCounters().OnFailedWriteResponse(reason); - return; - }; - - if (SpaceWatcher->SubDomainOutOfSpace && - (!record.HasModificationType() || (record.GetModificationType() != NKikimrTxColumnShard::TEvWrite::OPERATION_DELETE))) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_writing")("reason", "quota_exceeded"); - Counters.GetTabletCounters()->IncCounter(COUNTER_OUT_OF_SPACE); - return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::Overload, NKikimrTxColumnShard::EResultStatus::OVERLOADED); - } - - if (!AppDataVerified().ColumnShardConfig.GetWritingEnabled()) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "skip_writing")("reason", "disabled"); - return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::Disabled, NKikimrTxColumnShard::EResultStatus::ERROR); - } - - if (!TablesManager.IsReadyForStartWrite(pathId, false)) { - LOG_S_NOTICE("Write (fail) into pathId:" << writeMeta.GetTableId() << (TablesManager.HasPrimaryIndex() ? "" : " no index") - << " at tablet " << TabletID()); - - return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::NoTable, NKikimrTxColumnShard::EResultStatus::ERROR); - } - - { - auto status = TablesManager.GetPrimaryIndexAsVerified() - .GetGranuleVerified(writeMeta.GetTableId()) - .GetOptimizerPlanner() - .CheckWriteData(); - if (status.IsFail()) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "writing_fail_through_compaction")("reason", status.GetErrorMessage()); - return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::CompactionCriteria, NKikimrTxColumnShard::EResultStatus::ERROR); - } - } - - const auto& snapshotSchema = TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetLastSchema(); - auto arrowData = std::make_shared(snapshotSchema); - if (!arrowData->ParseFromProto(record)) { - LOG_S_ERROR( - "Write (fail) " << record.GetData().size() << " bytes into pathId " << writeMeta.GetTableId() << " at tablet " << TabletID()); - return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::IncorrectSchema, NKikimrTxColumnShard::EResultStatus::ERROR); - } - - NEvWrite::TWriteData writeData(writeMetaPtr, arrowData, snapshotSchema->GetIndexInfo().GetReplaceKey(), - StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING), false); - auto overloadStatus = CheckOverloadedImmediate(pathId); - if (overloadStatus == EOverloadStatus::None) { - overloadStatus = CheckOverloadedWait(pathId); - } - if (overloadStatus != EOverloadStatus::None) { - std::unique_ptr result = std::make_unique( - TabletID(), writeData.GetWriteMeta(), NKikimrTxColumnShard::EResultStatus::OVERLOADED); - OverloadWriteFail(overloadStatus, writeData.GetWriteMeta(), writeData.GetSize(), cookie, std::move(result), ctx); - Counters.GetCSCounters().OnFailedWriteResponse(EWriteFailReason::Overload); - } else { - if (ui64 writeId = (ui64)HasLongTxWrite(writeMeta.GetLongTxIdUnsafe(), writeMeta.GetWritePartId())) { - LOG_S_DEBUG("Write (duplicate) into pathId " << writeMeta.GetTableId() << " longTx " << writeMeta.GetLongTxIdUnsafe().ToString() - << " at tablet " << TabletID()); - - Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_DUPLICATE); - - auto result = - std::make_unique(TabletID(), writeMeta, writeId, NKikimrTxColumnShard::EResultStatus::SUCCESS); - ctx.Send(writeMeta.GetSource(), result.release()); - Counters.GetCSCounters().OnFailedWriteResponse(EWriteFailReason::LongTxDuplication); - return; - } - - Counters.GetWritesMonitor()->OnStartWrite(writeData.GetSize()); - - LOG_S_DEBUG("Write (blob) " << writeData.GetSize() << " bytes into pathId " << writeMeta.GetTableId() - << (writeMeta.GetWriteId() ? (" writeId " + ToString(writeMeta.GetWriteId())).c_str() : " ") - << Counters.GetWritesMonitor()->DebugString() << " at tablet " << TabletID()); - - NOlap::TWritingContext context(TabletID(), SelfId(), snapshotSchema, StoragesManager, Counters.GetIndexationCounters().SplitterCounters, - Counters.GetCSCounters().WritingCounters, GetLastTxSnapshot(), std::make_shared(1), true, - BufferizationInsertionWriteActorId, BufferizationPortionsWriteActorId); - std::shared_ptr task = std::make_shared(std::move(writeData), context); - NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); - } -} - class TCommitOperation { private: const ui64 TabletId; diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 485a4f09c056..73de964f7deb 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -257,7 +257,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void Handle(TEvColumnShard::TEvCancelTransactionProposal::TPtr& ev, const TActorContext& ctx); void Handle(TEvColumnShard::TEvNotifyTxCompletion::TPtr& ev, const TActorContext& ctx); void Handle(TEvTxProcessing::TEvPlanStep::TPtr& ev, const TActorContext& ctx); - void Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContext& ctx); void Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContext& ctx); void Handle(TEvColumnShard::TEvInternalScan::TPtr& ev, const TActorContext& ctx); void Handle(TEvMediatorTimecast::TEvRegisterTabletResult::TPtr& ev, const TActorContext& ctx); @@ -421,7 +420,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa HFunc(TEvDataShard::TEvKqpScan, Handle); HFunc(TEvColumnShard::TEvInternalScan, Handle); HFunc(TEvTxProcessing::TEvPlanStep, Handle); - HFunc(TEvColumnShard::TEvWrite, Handle); HFunc(TEvPrivate::TEvWriteBlobsResult, Handle); HFunc(TEvPrivate::TEvStartCompaction, Handle); HFunc(TEvPrivate::TEvMetadataAccessorsInfo, Handle); diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index 155fceb85af1..a47cfeb985dd 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -173,27 +173,6 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, waitResult ? &ids : nullptr, mType, lockId); } -std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, - ui64 tableId, const ui64 writePartId, const TString& data, - const std::vector& ydbSchema, const NEvWrite::EModificationType mType) -{ - auto write = std::make_unique(sender, longTxId, tableId, "0", data, writePartId, mType); - write->SetArrowSchema(NArrow::SerializeSchema(*NArrow::MakeArrowSchema(ydbSchema))); - - ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, write.release()); - TAutoPtr handle; - auto event = runtime.GrabEdgeEvent(handle); - UNIT_ASSERT(event); - - auto& resWrite = Proto(event); - UNIT_ASSERT_EQUAL(resWrite.GetOrigin(), TTestTxConfig::TxTablet0); - UNIT_ASSERT_EQUAL(resWrite.GetTxInitiator(), 0); - if (resWrite.GetStatus() == NKikimrTxColumnShard::EResultStatus::SUCCESS) { - return resWrite.GetWriteId(); - } - return {}; -} - void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vector& pathIds, NOlap::TSnapshot snap, ui64 scanId) { auto scan = std::make_unique(); diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 8a6f4ad05006..58216b25067c 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -439,10 +439,6 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, const std::vector& ydbSchema, bool waitResult = true, std::vector* writeIds = nullptr, const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert, const ui64 lockId = 1); -std::optional WriteData(TTestBasicRuntime& runtime, TActorId& sender, const NLongTxService::TLongTxId& longTxId, ui64 tableId, - const ui64 writePartId, const TString& data, const std::vector& ydbSchema, - const NEvWrite::EModificationType mType = NEvWrite::EModificationType::Upsert); - ui32 WaitWriteResult(TTestBasicRuntime& runtime, ui64 shardId, std::vector* writeIds = nullptr); void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vector& pathIds, NOlap::TSnapshot snap, ui64 scanId = 0); diff --git a/ydb/core/tx/data_events/columnshard_splitter.cpp b/ydb/core/tx/data_events/columnshard_splitter.cpp index 14a9a8eba5aa..efffd3f16570 100644 --- a/ydb/core/tx/data_events/columnshard_splitter.cpp +++ b/ydb/core/tx/data_events/columnshard_splitter.cpp @@ -72,8 +72,7 @@ NKikimr::NEvWrite::IShardsSplitter::TYdbConclusionStatus TColumnShardShardsSplit const TString schemaString = NArrow::SerializeSchema(*batch->schema()); for (auto&& [shardId, chunks] : split.GetResult()) { for (auto&& c : chunks) { - result.AddShardInfo(shardId, std::make_shared(schemaString, c.GetData(), c.GetRowsCount(), - sharding->GetShardInfoVerified(shardId).GetShardingVersion())); + result.AddShardInfo(shardId, std::make_shared(schemaString, c.GetData(), c.GetRowsCount())); } } diff --git a/ydb/core/tx/data_events/columnshard_splitter.h b/ydb/core/tx/data_events/columnshard_splitter.h index 2d0feda6b3a7..cb699fe1ce98 100644 --- a/ydb/core/tx/data_events/columnshard_splitter.h +++ b/ydb/core/tx/data_events/columnshard_splitter.h @@ -1,14 +1,14 @@ #pragma once #include "events.h" -#include "shards_splitter.h" #include "payload_helper.h" +#include "shards_splitter.h" -#include -#include -#include #include +#include #include +#include +#include namespace NKikimr::NEvWrite { @@ -18,13 +18,12 @@ class TColumnShardShardsSplitter: public IShardsSplitter { const TString SchemaData; const TString Data; const ui32 RowsCount; - const ui32 GranuleShardingVersion; + public: - TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount, const ui32 granuleShardingVersion) + TShardInfo(const TString& schemaData, const TString& data, const ui32 rowsCount) : SchemaData(schemaData) , Data(data) - , RowsCount(rowsCount) - , GranuleShardingVersion(granuleShardingVersion) { + , RowsCount(rowsCount) { } virtual ui64 GetBytes() const override { @@ -39,10 +38,6 @@ class TColumnShardShardsSplitter: public IShardsSplitter { return Data; } - virtual void Serialize(TEvColumnShard::TEvWrite& evWrite) const override { - evWrite.SetArrowData(SchemaData, Data); - evWrite.Record.SetGranuleShardingVersion(GranuleShardingVersion); - } virtual void Serialize(NEvents::TDataEvents::TEvWrite& evWrite, const ui64 tableId, const ui64 schemaVersion) const override { TPayloadWriter writer(evWrite); TString data = Data; @@ -66,4 +61,4 @@ class TColumnShardShardsSplitter: public IShardsSplitter { std::shared_ptr ExtractArrowSchema(const NKikimrSchemeOp::TColumnTableSchema& schema); }; -} +} // namespace NKikimr::NEvWrite diff --git a/ydb/core/tx/data_events/shard_writer.cpp b/ydb/core/tx/data_events/shard_writer.cpp index 1ce27af0d670..a31f48db69d4 100644 --- a/ydb/core/tx/data_events/shard_writer.cpp +++ b/ydb/core/tx/data_events/shard_writer.cpp @@ -8,10 +8,9 @@ namespace NKikimr::NEvWrite { - TWritersController::TWritersController(const ui32 writesCount, const NActors::TActorIdentity& longTxActorId, const NLongTxService::TLongTxId& longTxId, const bool immediateWrite) + TWritersController::TWritersController(const ui32 writesCount, const NActors::TActorIdentity& longTxActorId, const NLongTxService::TLongTxId& longTxId) : WritesCount(writesCount) , LongTxActorId(longTxActorId) - , ImmediateWrite(immediateWrite) , LongTxId(longTxId) { Y_ABORT_UNLESS(writesCount); @@ -40,7 +39,7 @@ namespace NKikimr::NEvWrite { } TShardWriter::TShardWriter(const ui64 shardId, const ui64 tableId, const ui64 schemaVersion, const TString& dedupId, const IShardInfo::TPtr& data, - const NWilson::TProfileSpan& parentSpan, TWritersController::TPtr externalController, const ui32 writePartIdx, const EModificationType mType, const bool immediateWrite, + const NWilson::TProfileSpan& parentSpan, TWritersController::TPtr externalController, const ui32 writePartIdx, const std::optional timeout ) : ShardId(shardId) @@ -52,22 +51,14 @@ namespace NKikimr::NEvWrite { , ExternalController(externalController) , LeaderPipeCache(MakePipePerNodeCacheID(false)) , ActorSpan(parentSpan.BuildChildrenSpan("ShardWriter")) - , ModificationType(mType) - , ImmediateWrite(immediateWrite) , Timeout(timeout) { } void TShardWriter::SendWriteRequest() { - if (ImmediateWrite) { - auto ev = MakeHolder(NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE); - DataForShard->Serialize(*ev, TableId, SchemaVersion); - SendToTablet(std::move(ev)); - } else { - auto ev = MakeHolder(SelfId(), ExternalController->GetLongTxId(), TableId, DedupId, "", WritePartIdx, ModificationType); - DataForShard->Serialize(*ev); - SendToTablet(std::move(ev)); - } + auto ev = MakeHolder(NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE); + DataForShard->Serialize(*ev, TableId, SchemaVersion); + SendToTablet(std::move(ev)); } void TShardWriter::Bootstrap() { @@ -101,28 +92,6 @@ namespace NKikimr::NEvWrite { ExternalController->OnSuccess(ShardId, 0, WritePartIdx); } - void TShardWriter::Handle(TEvColumnShard::TEvWriteResult::TPtr& ev) { - const auto* msg = ev->Get(); - Y_ABORT_UNLESS(msg->Record.GetOrigin() == ShardId); - - const auto ydbStatus = msg->GetYdbStatus(); - if (ydbStatus == Ydb::StatusIds::OVERLOADED) { - if (RetryWriteRequest(true)) { - return; - } - } - - auto gPassAway = PassAwayGuard(); - if (ydbStatus != Ydb::StatusIds::SUCCESS) { - ExternalController->OnFail(ydbStatus, - TStringBuilder() << "Cannot write data into shard " << ShardId << " in longTx " << - ExternalController->GetLongTxId().ToString()); - return; - } - - ExternalController->OnSuccess(ShardId, msg->Record.GetWriteId(), WritePartIdx); - } - void TShardWriter::Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev) { NWilson::TProfileSpan pSpan(0, ActorSpan.GetTraceId(), "DeliveryProblem"); const auto* msg = ev->Get(); diff --git a/ydb/core/tx/data_events/shard_writer.h b/ydb/core/tx/data_events/shard_writer.h index 595d61e7c5e2..8bf1cfbdc259 100644 --- a/ydb/core/tx/data_events/shard_writer.h +++ b/ydb/core/tx/data_events/shard_writer.h @@ -113,7 +113,6 @@ class TWritersController { NActors::TActorIdentity LongTxActorId; std::vector WriteIds; const TMonotonic StartInstant = TMonotonic::Now(); - const bool ImmediateWrite = false; YDB_READONLY_DEF(NLongTxService::TLongTxId, LongTxId); YDB_READONLY(std::shared_ptr, Counters, std::make_shared()); void SendReply() { @@ -121,16 +120,9 @@ class TWritersController { Counters->OnFailedFullReply(TMonotonic::Now() - StartInstant); AFL_VERIFY(Code); LongTxActorId.Send(LongTxActorId, new TEvPrivate::TEvShardsWriteResult(*Code, Issues)); - } else if (ImmediateWrite) { - Counters->OnSucceedFullReply(TMonotonic::Now() - StartInstant); - LongTxActorId.Send(LongTxActorId, new TEvPrivate::TEvShardsWriteResult(Ydb::StatusIds::SUCCESS)); } else { Counters->OnSucceedFullReply(TMonotonic::Now() - StartInstant); - auto req = MakeHolder(LongTxId); - for (auto&& i : WriteIds) { - req->AddWrite(i.GetShardId(), i.GetWriteId()); - } - LongTxActorId.Send(NLongTxService::MakeLongTxServiceID(LongTxActorId.NodeId()), req.Release()); + LongTxActorId.Send(LongTxActorId, new TEvPrivate::TEvShardsWriteResult(Ydb::StatusIds::SUCCESS)); } } @@ -157,8 +149,7 @@ class TWritersController { }; }; - TWritersController(const ui32 writesCount, const NActors::TActorIdentity& longTxActorId, const NLongTxService::TLongTxId& longTxId, - const bool immediateWrite); + TWritersController(const ui32 writesCount, const NActors::TActorIdentity& longTxActorId, const NLongTxService::TLongTxId& longTxId); void OnSuccess(const ui64 shardId, const ui64 writeId, const ui32 writePartId); void OnFail(const Ydb::StatusIds::StatusCode code, const TString& message); }; @@ -180,8 +171,6 @@ class TShardWriter: public NActors::TActorBootstrapped { TWritersController::TPtr ExternalController; const TActorId LeaderPipeCache; NWilson::TProfileSpan ActorSpan; - EModificationType ModificationType; - const bool ImmediateWrite = false; const std::optional Timeout; void SendWriteRequest(); @@ -200,11 +189,10 @@ class TShardWriter: public NActors::TActorBootstrapped { public: TShardWriter(const ui64 shardId, const ui64 tableId, const ui64 schemaVersion, const TString& dedupId, const IShardInfo::TPtr& data, const NWilson::TProfileSpan& parentSpan, TWritersController::TPtr externalController, const ui32 writePartIdx, - const EModificationType mType, const bool immediateWrite, const std::optional timeout = std::nullopt); + const std::optional timeout = std::nullopt); STFUNC(StateMain) { switch (ev->GetTypeRewrite()) { - hFunc(TEvColumnShard::TEvWriteResult, Handle); hFunc(TEvPipeCache::TEvDeliveryProblem, Handle); hFunc(NEvents::TDataEvents::TEvWriteResult, Handle); hFunc(NActors::TEvents::TEvWakeup, Handle); @@ -214,7 +202,6 @@ class TShardWriter: public NActors::TActorBootstrapped { void Bootstrap(); void Handle(NActors::TEvents::TEvWakeup::TPtr& ev); - void Handle(TEvColumnShard::TEvWriteResult::TPtr& ev); void Handle(TEvPipeCache::TEvDeliveryProblem::TPtr& ev); void Handle(NEvents::TDataEvents::TEvWriteResult::TPtr& ev); diff --git a/ydb/core/tx/data_events/shards_splitter.h b/ydb/core/tx/data_events/shards_splitter.h index bde0656ccb91..f6b977f4df9c 100644 --- a/ydb/core/tx/data_events/shards_splitter.h +++ b/ydb/core/tx/data_events/shards_splitter.h @@ -40,7 +40,6 @@ class IShardsSplitter { using TPtr = std::shared_ptr; virtual ~IShardInfo() {} - virtual void Serialize(TEvColumnShard::TEvWrite& evWrite) const = 0; virtual void Serialize(NEvents::TDataEvents::TEvWrite& evWrite, const ui64 tableId, const ui64 schemaVersion) const = 0; virtual ui64 GetBytes() const = 0; virtual ui32 GetRowsCount() const = 0; diff --git a/ydb/core/tx/tx_proxy/rpc_long_tx.cpp b/ydb/core/tx/tx_proxy/rpc_long_tx.cpp index 24f0d1a8bae6..4a24e86d582e 100644 --- a/ydb/core/tx/tx_proxy/rpc_long_tx.cpp +++ b/ydb/core/tx/tx_proxy/rpc_long_tx.cpp @@ -2,13 +2,13 @@ #include #include -#include #include #include #include #include #include +#include #include #include @@ -29,13 +29,10 @@ class TLongTxWriteBase: public TActorBootstrapped, protected: using TThis = typename TBase::TThis; - const bool NoTxWrite = false; public: - TLongTxWriteBase(const TString& databaseName, const TString& path, const TString& token, const TLongTxId& longTxId, const TString& dedupId, - const bool noTxWrite) - : NoTxWrite(noTxWrite) - , DatabaseName(databaseName) + TLongTxWriteBase(const TString& databaseName, const TString& path, const TString& token, const TLongTxId& longTxId, const TString& dedupId) + : DatabaseName(databaseName) , Path(path) , DedupId(dedupId) , LongTxId(longTxId) @@ -97,8 +94,7 @@ class TLongTxWriteBase: public TActorBootstrapped, const auto& splittedData = shardsSplitter->GetSplitData(); const auto& shardsInRequest = splittedData.GetShardRequestsCount(); - InternalController = - std::make_shared(shardsInRequest, this->SelfId(), LongTxId, NoTxWrite); + InternalController = std::make_shared(shardsInRequest, this->SelfId(), LongTxId); InternalController->GetCounters()->OnSplitByShards(shardsInRequest); ui32 sumBytes = 0; @@ -109,9 +105,8 @@ class TLongTxWriteBase: public TActorBootstrapped, InternalController->GetCounters()->OnRequest(shardInfo->GetRowsCount(), shardInfo->GetBytes()); sumBytes += shardInfo->GetBytes(); rowsCount += shardInfo->GetRowsCount(); - this->Register( - new NEvWrite::TShardWriter(shard, shardsSplitter->GetTableId(), shardsSplitter->GetSchemaVersion(), DedupId, shardInfo, - ActorSpan, InternalController, ++writeIdx, NEvWrite::EModificationType::Replace, NoTxWrite, TDuration::Seconds(20))); + this->Register(new NEvWrite::TShardWriter(shard, shardsSplitter->GetTableId(), shardsSplitter->GetSchemaVersion(), DedupId, + shardInfo, ActorSpan, InternalController, ++writeIdx, TDuration::Seconds(20))); } } pSpan.Attribute("affected_shards_count", (long)splittedData.GetShardsInfo().size()); @@ -235,8 +230,8 @@ class TLongTxWriteInternal: public TLongTxWriteBase { public: explicit TLongTxWriteInternal(const TActorId& replyTo, const TLongTxId& longTxId, const TString& dedupId, const TString& databaseName, const TString& path, std::shared_ptr navigateResult, std::shared_ptr batch, - std::shared_ptr issues, const bool noTxWrite) - : TBase(databaseName, path, TString(), longTxId, dedupId, noTxWrite) + std::shared_ptr issues) + : TBase(databaseName, path, TString(), longTxId, dedupId) , ReplyTo(replyTo) , NavigateResult(navigateResult) , Batch(batch) @@ -283,9 +278,8 @@ class TLongTxWriteInternal: public TLongTxWriteBase { TActorId DoLongTxWriteSameMailbox(const TActorContext& ctx, const TActorId& replyTo, const NLongTxService::TLongTxId& longTxId, const TString& dedupId, const TString& databaseName, const TString& path, std::shared_ptr navigateResult, std::shared_ptr batch, - std::shared_ptr issues, const bool noTxWrite) { - return ctx.RegisterWithSameMailbox( - new TLongTxWriteInternal(replyTo, longTxId, dedupId, databaseName, path, navigateResult, batch, issues, noTxWrite)); + std::shared_ptr issues) { + return ctx.RegisterWithSameMailbox(new TLongTxWriteInternal(replyTo, longTxId, dedupId, databaseName, path, navigateResult, batch, issues)); } // diff --git a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h index df317ee5b39c..73d6e8580c7d 100644 --- a/ydb/core/tx/tx_proxy/upload_rows_common_impl.h +++ b/ydb/core/tx/tx_proxy/upload_rows_common_impl.h @@ -106,7 +106,7 @@ TActorId DoLongTxWriteSameMailbox(const TActorContext& ctx, const TActorId& repl const NLongTxService::TLongTxId& longTxId, const TString& dedupId, const TString& databaseName, const TString& path, std::shared_ptr navigateResult, std::shared_ptr batch, - std::shared_ptr issues, const bool noTxWrite); + std::shared_ptr issues); template class TUploadRowsBase : public TActorBootstrapped> { @@ -148,7 +148,6 @@ class TUploadRowsBase : public TActorBootstrappedNow(); - ImmediateWrite = AppData(ctx)->FeatureFlags.GetEnableImmediateWritingOnBulkUpsert(); OnBeforeStart(ctx); ResolveTable(GetTable(), ctx); } @@ -889,7 +887,7 @@ class TUploadRowsBase : public TActorBootstrapped Date: Thu, 12 Jun 2025 13:49:51 +0300 Subject: [PATCH 16/62] Remap path (#15991) Co-authored-by: Evgeny Zverev --- ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp | 19 +- ydb/core/kqp/ut/olap/indexes_ut.cpp | 27 ++- ydb/core/kqp/ut/olap/tiering_ut.cpp | 14 +- .../transaction/tx_blobs_written.cpp | 10 +- .../blobs_action/transaction/tx_write.cpp | 8 +- ydb/core/tx/columnshard/columnshard.cpp | 11 +- ydb/core/tx/columnshard/columnshard.h | 10 +- ydb/core/tx/columnshard/columnshard__scan.cpp | 19 +- .../columnshard/columnshard__statistics.cpp | 5 +- .../tx/columnshard/columnshard__write.cpp | 16 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 43 ++-- ydb/core/tx/columnshard/columnshard_schema.h | 8 +- ydb/core/tx/columnshard/common/path_id.cpp | 187 +++++++++++++++++- ydb/core/tx/columnshard/common/path_id.h | 73 ++++++- ydb/core/tx/columnshard/common/ya.make | 4 + .../tx/columnshard/counters/column_tables.cpp | 2 +- .../tx/columnshard/counters/column_tables.h | 2 +- .../destination/events/transfer.h | 2 +- .../columnshard/engines/column_engine_logs.h | 4 - .../engines/portions/portion_info.cpp | 4 +- .../engines/reader/abstract/read_metadata.cpp | 6 +- .../engines/reader/common/description.h | 2 +- .../constructor/read_metadata.cpp | 8 +- .../common_reader/constructor/read_metadata.h | 4 +- .../reader/sys_view/abstract/filler.cpp | 23 +-- .../engines/reader/sys_view/abstract/filler.h | 8 +- .../reader/sys_view/abstract/granule_view.h | 8 +- .../engines/reader/sys_view/chunks/chunks.cpp | 8 +- .../engines/reader/sys_view/chunks/chunks.h | 2 +- .../reader/sys_view/granules/granules.cpp | 2 +- .../reader/sys_view/optimizer/optimizer.cpp | 2 +- .../reader/sys_view/optimizer/optimizer.h | 11 +- .../reader/sys_view/portions/portions.cpp | 7 +- .../reader/sys_view/portions/portions.h | 2 +- .../reader/transaction/tx_internal_scan.cpp | 2 +- .../engines/reader/transaction/tx_scan.cpp | 7 +- .../engines/storage/granule/storage.h | 13 -- .../engines/writer/buffer/actor2.cpp | 2 +- .../writer/indexed_blob_constructor.cpp | 2 +- .../columnshard/export/common/identifier.cpp | 4 +- .../tx/columnshard/hooks/abstract/abstract.h | 11 ++ .../columnshard/hooks/testing/controller.cpp | 17 -- .../tx/columnshard/hooks/testing/controller.h | 82 +++++++- .../columnshard/hooks/testing/ro_controller.h | 4 + .../operations/batch_builder/restore.cpp | 4 +- .../operations/batch_builder/restore.h | 1 - ydb/core/tx/columnshard/operations/events.h | 4 +- .../tx/columnshard/operations/manager.cpp | 4 +- ydb/core/tx/columnshard/operations/manager.h | 2 +- .../operations/slice_builder/builder.cpp | 4 +- ydb/core/tx/columnshard/operations/write.cpp | 19 +- ydb/core/tx/columnshard/operations/write.h | 4 +- ydb/core/tx/columnshard/tables_manager.cpp | 71 ++++++- ydb/core/tx/columnshard/tables_manager.h | 37 ++-- .../tx/columnshard/tablet/write_queue.cpp | 8 +- ydb/core/tx/columnshard/tablet/write_queue.h | 8 +- .../test_helper/columnshard_ut_common.cpp | 2 +- .../test_helper/columnshard_ut_common.h | 11 +- .../transactions/operators/schema.cpp | 8 +- .../transactions/operators/schema.h | 10 +- .../tx/columnshard/ut_rw/ut_normalizer.cpp | 6 +- ydb/core/tx/data_events/write_data.h | 6 +- 62 files changed, 668 insertions(+), 246 deletions(-) diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index cb222d0a3483..498412b9370d 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -111,7 +111,8 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { AFL_VERIFY(ShardIds.size() == ShardsCount)("count", ShardIds.size())("ids", JoinSeq(",", ShardIds)); std::set pathIdsSet; for (auto&& i : ShardIds) { - auto pathIds = Controller->GetPathIds(i); + const auto pathIdTranslator = Controller->GetPathIdTranslator(i); + auto pathIds = pathIdTranslator->GetInternalPathIds(); pathIdsSet.insert(pathIds.begin(), pathIds.end()); } PathIds = std::vector(pathIdsSet.begin(), pathIdsSet.end()); @@ -502,17 +503,17 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { // tester.CheckCount(); // } - Y_UNIT_TEST(SplitEmpty) { - TAsyncReshardingTest tester; + // Y_UNIT_TEST(SplitEmpty) { + // TAsyncReshardingTest tester; - tester.CheckCount(); + // tester.CheckCount(); - tester.StartResharding("SPLIT"); + // tester.StartResharding("SPLIT"); - tester.CheckCount(); - tester.WaitResharding(); - tester.CheckCount(); - } + // tester.CheckCount(); + // tester.WaitResharding(); + // tester.CheckCount(); + // } // Y_UNIT_TEST(ChangeSchemaAndSplit) { // TAsyncReshardingTest tester; diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index 333da41e340d..8152ccba8c83 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -246,28 +246,25 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { TAutoPtr handle; - size_t shard = 0; - std::set pathids; + std::optional schemeShardLocalPathId; for (auto&& i : csController->GetShardActualIds()) { Cerr << ">>> shard actual id: " << i << Endl; - for (auto&& j : csController->GetPathIds(i)) { - Cerr << ">>> path id: " << j << Endl; - pathids.insert(j); - } - if (++shard == 3) { - break; + const auto pathIds = csController->GetPathIdTranslator(i)->GetSchemeShardLocalPathIds(); + UNIT_ASSERT(pathIds.size() == 1); + if (schemeShardLocalPathId.has_value()) { + UNIT_ASSERT(schemeShardLocalPathId == *pathIds.begin()); + } else { + schemeShardLocalPathId = *pathIds.begin(); } } - UNIT_ASSERT(pathids.size() == 1); - const auto& pathId = *pathids.begin(); + UNIT_ASSERT(schemeShardLocalPathId.has_value()); - shard = 0; - for (auto&& i : csController->GetShardActualIds()) { + size_t shard = 0; + for (const auto& [tabletId, pathIdTranslator]: csController->GetActiveTablets()) { auto request = std::make_unique(); - request->Record.MutableTable()->MutablePathId()->SetLocalId(pathId.GetRawValue()); - - runtime->Send(MakePipePerNodeCacheID(false), sender, new TEvPipeCache::TEvForward(request.release(), i, false)); + request->Record.MutableTable()->MutablePathId()->SetLocalId(schemeShardLocalPathId->GetRawValue()); + runtime->Send(MakePipePerNodeCacheID(false), sender, new TEvPipeCache::TEvForward(request.release(), static_cast(tabletId), false)); if (++shard == 3) { break; } diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index ce60f8ae288f..401ee5874353 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -335,13 +335,13 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { olapHelper.CreateTestOlapTable(); tieringHelper.WriteSampleData(); csController->WaitCompactions(TDuration::Seconds(5)); - THashSet pathsToLock{ - NColumnShard::TInternalPathId::FromRawValue(0), - NColumnShard::TInternalPathId::FromRawValue(1), - NColumnShard::TInternalPathId::FromRawValue(2), - NColumnShard::TInternalPathId::FromRawValue(3), - NColumnShard::TInternalPathId::FromRawValue(4), - NColumnShard::TInternalPathId::FromRawValue(5), + THashSet pathsToLock; + for (const auto& [_, pathIdTranslator]: csController->GetActiveTablets()) { + for (size_t i = 0; i != 6; ++i) { + if (auto internalPathId = pathIdTranslator->ResolveInternalPathId(NColumnShard::TSchemeShardLocalPathId::FromRawValue(i))) { + pathsToLock.insert(*internalPathId); + } + } }; csController->RegisterLock("table", std::make_shared("table", std::move(pathsToLock), NOlap::NDataLocks::ELockCategory::Compaction)); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp index 6c611d48e0a9..c054cc25bb33 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp @@ -57,7 +57,7 @@ bool TTxBlobsWritingFinished::DoExecute(TTransactionContext& txc, const TActorCo if (!operationIds.emplace(operation->GetWriteId()).second) { continue; } - AFL_VERIFY(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetTableId(), minReadSnapshot)); + AFL_VERIFY(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetPathId().InternalPathId, minReadSnapshot)); Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { operation->OnWriteFinish(txc, {}, true); @@ -75,7 +75,7 @@ bool TTxBlobsWritingFinished::DoExecute(TTransactionContext& txc, const TActorCo lock.SetDataShard(Self->TabletID()); lock.SetGeneration(info.GetGeneration()); lock.SetCounter(info.GetInternalGenerationCounter()); - lock.SetPathId(writeMeta.GetTableId().GetRawValue()); + writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(lock); auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); } @@ -103,11 +103,11 @@ void TTxBlobsWritingFinished::DoComplete(const TActorContext& ctx) { continue; } auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - pathIds.emplace(op->GetPathId()); + pathIds.emplace(op->GetPathId().InternalPathId); if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetTableId())) { + if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetPathId().InternalPathId)) { auto evWrite = std::make_shared( - writeMeta.GetTableId(), writeResult.GetPKBatchVerified(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); + writeMeta.GetPathId().InternalPathId, writeResult.GetPKBatchVerified(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); } } diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index bdcb27862bbe..c25c44196197 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -43,7 +43,7 @@ bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { const auto minReadSnapshot = Self->GetMinReadSnapshot(); for (auto&& aggr : buffer.GetAggregations()) { const auto& writeMeta = aggr->GetWriteMeta(); - Y_ABORT_UNLESS(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetTableId(), minReadSnapshot)); + Y_ABORT_UNLESS(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetPathId().InternalPathId, minReadSnapshot)); txc.DB.NoMoreReadsForTx(); TWriteOperation::TPtr operation; AFL_VERIFY(!writeMeta.HasLongTxId()); @@ -90,7 +90,7 @@ bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { lock.SetDataShard(Self->TabletID()); lock.SetGeneration(info.GetGeneration()); lock.SetCounter(info.GetInternalGenerationCounter()); - lock.SetPathId(writeMeta.GetTableId().GetRawValue()); + writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(lock); auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); } @@ -120,8 +120,8 @@ void TTxWrite::DoComplete(const TActorContext& ctx) { AFL_VERIFY(!writeMeta.HasLongTxId()); auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetTableId())) { - auto evWrite = std::make_shared(writeMeta.GetTableId(), + if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetPathId().InternalPathId)) { + auto evWrite = std::make_shared(writeMeta.GetPathId().InternalPathId, buffer.GetAggregations()[i]->GetRecordBatch(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); } diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 81a4bf074355..daa6ce49192b 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -85,7 +85,7 @@ void TColumnShard::TrySwitchToWork(const TActorContext& ctx) { Counters.GetCSCounters().OnIndexMetadataLimit(NOlap::IColumnEngine::GetMetadataLimit()); EnqueueBackgroundActivities(); BackgroundSessionsManager->Start(); - ctx.Send(SelfId(), new NActors::TEvents::TEvWakeup()); + ctx.Send(SelfId(), new NActors::TEvents::TEvWakeup()); ctx.Send(SelfId(), new TEvPrivate::TEvPeriodicWakeup()); ctx.Send(SelfId(), new TEvPrivate::TEvPingSnapshotsUsage()); NYDBTest::TControllers::GetColumnShardController()->OnSwitchToWork(TabletID()); @@ -411,10 +411,11 @@ void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_pt TTableStatsBuilder tableStatsBuilder(Counters, Executor()); LOG_S_DEBUG("There are stats for " << tables.size() << " tables"); - for (const auto& [pathId, _] : tables) { + for (const auto& [internalPathId, table] : tables) { + const auto& schemeShardLocalPathId = table.GetPathId().SchemeShardLocalPathId; auto* periodicTableStats = ev->Record.AddTables(); periodicTableStats->SetDatashardId(TabletID()); - periodicTableStats->SetTableLocalId(pathId.GetRawValue()); + periodicTableStats->SetTableLocalId(schemeShardLocalPathId.GetRawValue()); periodicTableStats->SetShardState(2); // NKikimrTxDataShard.EDatashardState.Ready periodicTableStats->SetGeneration(Executor()->Generation()); @@ -426,9 +427,9 @@ void TColumnShard::FillColumnTableStats(const TActorContext& ctx, std::unique_pt resourceMetrics->Fill(*periodicTableStats->MutableTabletMetrics()); } - tableStatsBuilder.FillTableStats(pathId, *(periodicTableStats->MutableTableStats())); + tableStatsBuilder.FillTableStats(internalPathId, *(periodicTableStats->MutableTableStats())); - LOG_S_TRACE("Add stats for table, tableLocalID=" << pathId); + LOG_S_TRACE("Add stats for table, tableLocalID=" << schemeShardLocalPathId); } } diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h index 1e45af970e3e..fefa2c5a62c1 100644 --- a/ydb/core/tx/columnshard/columnshard.h +++ b/ydb/core/tx/columnshard/columnshard.h @@ -98,7 +98,7 @@ namespace TEvColumnShard { struct TEvInternalScan: public TEventLocal { private: - YDB_READONLY_DEF(NColumnShard::TInternalPathId, PathId); + YDB_READONLY_DEF(NColumnShard::TUnifiedPathId, PathId); YDB_READONLY(NOlap::TSnapshot, Snapshot, NOlap::TSnapshot::Zero()); YDB_READONLY_DEF(std::optional, LockId); YDB_ACCESSOR(bool, Reverse, false); @@ -114,7 +114,7 @@ namespace TEvColumnShard { ColumnIds.emplace_back(id); } - TEvInternalScan(const NColumnShard::TInternalPathId pathId, const NOlap::TSnapshot& snapshot, const std::optional lockId) + TEvInternalScan(const NColumnShard::TUnifiedPathId pathId, const NOlap::TSnapshot& snapshot, const std::optional lockId) : PathId(pathId) , Snapshot(snapshot) , LockId(lockId) @@ -242,11 +242,11 @@ namespace TEvColumnShard { struct TEvWrite : public TEventPB { TEvWrite() = default; - TEvWrite(const TActorId& source, const NLongTxService::TLongTxId& longTxId, ui64 tableId, + TEvWrite(const TActorId& source, const NLongTxService::TLongTxId& longTxId, NColumnShard::TSchemeShardLocalPathId tableId, const TString& dedupId, const TString& data, const ui32 writePartId, const NEvWrite::EModificationType modificationType) { ActorIdToProto(source, Record.MutableSource()); - Record.SetTableId(tableId); + tableId.ToProto(Record); Record.SetDedupId(dedupId); Record.SetData(data); Record.SetWritePartId(writePartId); @@ -279,7 +279,7 @@ namespace TEvColumnShard { Record.SetOrigin(origin); Record.SetTxInitiator(0); Record.SetWriteId(writeId); - Record.SetTableId(writeMeta.GetTableId().GetRawValue()); + writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(Record); Record.SetDedupId(writeMeta.GetDedupId()); Record.SetStatus(status); } diff --git a/ydb/core/tx/columnshard/columnshard__scan.cpp b/ydb/core/tx/columnshard/columnshard__scan.cpp index 1f0830dad1b5..ce6758f02d0d 100644 --- a/ydb/core/tx/columnshard/columnshard__scan.cpp +++ b/ydb/core/tx/columnshard/columnshard__scan.cpp @@ -1,6 +1,8 @@ #include "columnshard__scan.h" +#include #include "columnshard.h" #include "columnshard_impl.h" +#include "engines/reader/sys_view/abstract/policy.h" #include "engines/reader/transaction/tx_scan.h" #include "engines/reader/transaction/tx_internal_scan.h" @@ -30,7 +32,22 @@ void TColumnShard::Handle(TEvDataShard::TEvKqpScan::TPtr& ev, const TActorContex return; } - Counters.GetColumnTablesCounters()->GetPathIdCounter(TInternalPathId::FromRawValue(record.GetLocalPathId()))->OnReadEvent(); + const auto schemeShardLocalPath = TSchemeShardLocalPathId::FromProto(record); + auto internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPath); + if (!internalPathId && NOlap::NReader::NSysView::NAbstract::ISysViewPolicy::BuildByPath(record.GetTablePath())) { + internalPathId = TInternalPathId::FromRawValue(schemeShardLocalPath.GetRawValue()); //TODO register ColumnStore in tablesmanager + } + if (!internalPathId) { + const auto& request = ev->Get()->Record; + auto error = MakeHolder(request.GetGeneration(), TabletID()); + error->Record.SetStatus(Ydb::StatusIds::BAD_REQUEST); + auto issue = NYql::YqlIssue({}, NYql::TIssuesIds::KIKIMR_BAD_REQUEST,TStringBuilder() << "table: " << request.GetTablePath() << "not found"); + NYql::IssueToMessage(issue, error->Record.MutableIssues()->Add()); + + ctx.Send(ev->Sender, error.Release()); + return; + } + Counters.GetColumnTablesCounters()->GetPathIdCounter(*internalPathId)->OnReadEvent(); ScanTxInFlight.insert({txId, TAppData::TimeProvider->Now()}); Counters.GetTabletCounters()->SetCounter(COUNTER_SCAN_IN_FLY, ScanTxInFlight.size()); Execute(new NOlap::NReader::TTxScan(this, ev), ctx); diff --git a/ydb/core/tx/columnshard/columnshard__statistics.cpp b/ydb/core/tx/columnshard/columnshard__statistics.cpp index 63d3e54865d3..de1096342a1d 100644 --- a/ydb/core/tx/columnshard/columnshard__statistics.cpp +++ b/ydb/core/tx/columnshard/columnshard__statistics.cpp @@ -287,8 +287,11 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev, } AFL_VERIFY(HasIndex()); + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(record.GetTable().GetPathId()); + const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId); + AFL_VERIFY(internalPathId); auto index = GetIndexAs(); - auto spg = index.GetGranuleOptional(TInternalPathId::FromRawValue(record.GetTable().GetPathId().GetLocalId())); + auto spg = index.GetGranuleOptional(*internalPathId); AFL_VERIFY(spg); std::set columnTagsRequested; diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 12967588a2b6..014ad139c9c4 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -51,7 +51,7 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const Y_ABORT("invalid function usage"); } - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "write_overload")("size", writeSize)("path_id", writeMeta.GetTableId())( + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "write_overload")("size", writeSize)("path_id", writeMeta.GetPathId())( "reason", overloadReason); ctx.Send(writeMeta.GetSource(), event.release(), 0, cookie); @@ -162,7 +162,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo } else { const TMonotonic now = TMonotonic::Now(); Counters.OnWritePutBlobsSuccess(now - writeMeta.GetWriteStartInstant(), aggr->GetRows()); - LOG_S_DEBUG("Write (record) into pathId " << writeMeta.GetTableId() + LOG_S_DEBUG("Write (record) into pathId " << writeMeta.GetPathId() << (writeMeta.GetWriteId() ? (" writeId " + ToString(writeMeta.GetWriteId())).c_str() : "") << " at tablet " << TabletID()); } @@ -438,14 +438,16 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor return; } - const auto pathId = TInternalPathId::FromRawValue(operation.GetTableId().GetTableId()); - - if (!TablesManager.IsReadyForStartWrite(pathId, false)) { + const auto schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(operation.GetTableId()); + const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId); + AFL_VERIFY(internalPathId); + const auto& pathId = TUnifiedPathId{*internalPathId, schemeShardLocalPathId}; + if (!TablesManager.IsReadyForStartWrite(*internalPathId, false)) { sendError("table not writable", NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR); return; } - Counters.GetColumnTablesCounters()->GetPathIdCounter(pathId)->OnWriteEvent(); + Counters.GetColumnTablesCounters()->GetPathIdCounter(*internalPathId)->OnWriteEvent(); auto arrowData = std::make_shared(schema); if (!arrowData->Parse(operation, NEvWrite::TPayloadReader(*ev->Get()))) { @@ -462,7 +464,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor if (outOfSpace) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_writing")("reason", "quota_exceeded")("source", "dataevent"); } - auto overloadStatus = outOfSpace ? EOverloadStatus::Disk : CheckOverloadedImmediate(pathId); + auto overloadStatus = outOfSpace ? EOverloadStatus::Disk : CheckOverloadedImmediate(*internalPathId); if (overloadStatus != EOverloadStatus::None) { std::unique_ptr result = NEvents::TDataEvents::TEvWriteResult::BuildError( TabletID(), 0, NKikimrDataEvents::TEvWriteResult::STATUS_OVERLOADED, "overload data error"); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 3e4ec1b62678..8e34f5f7a0b5 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -379,18 +379,20 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl NTabletFlatExecutor::TTransactionContext& txc) { NIceDb::TNiceDb db(txc.DB); - const TInternalPathId pathId = TInternalPathId::FromRawValue(tableProto.GetPathId()); - if (TablesManager.HasTable(pathId)) { - LOG_S_DEBUG("EnsureTable for existed pathId: " << pathId << " at tablet " << TabletID()); + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(tableProto); + if (const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId)) { + LOG_S_DEBUG("EnsureTable for existed pathId: " << TUnifiedPathId(*internalPathId, schemeShardLocalPathId) + << " at tablet " << TabletID()); return; } + const auto internalPathId = TablesManager.CreateInternalPathId(schemeShardLocalPathId); - LOG_S_DEBUG("EnsureTable for pathId: " << pathId + LOG_S_DEBUG("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId) << " ttl settings: " << tableProto.GetTtlSettings() << " at tablet " << TabletID()); NKikimrTxColumnShard::TTableVersionInfo tableVerProto; - tableVerProto.SetPathId(pathId.GetRawValue()); + internalPathId.ToProto(tableVerProto); // check schema changed @@ -413,7 +415,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl { THashSet usedTiers; - TTableInfo table(pathId); + TTableInfo table({internalPathId, schemeShardLocalPathId}); if (tableProto.HasTtlSettings()) { const auto& ttlSettings = tableProto.GetTtlSettings(); *tableVerProto.MutableTtlSettings() = ttlSettings; @@ -423,14 +425,14 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl } TablesManager.RegisterTable(std::move(table), db); if (!usedTiers.empty()) { - ActivateTiering(pathId, usedTiers); + ActivateTiering(internalPathId, usedTiers); } } tableVerProto.SetSchemaPresetVersionAdj(tableProto.GetSchemaPresetVersionAdj()); - TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db); - InsertTable->RegisterPathInfo(pathId); + TablesManager.AddTableVersion(internalPathId, version, tableVerProto, schema, db); + InsertTable->RegisterPathInfo(internalPathId); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, TablesManager.GetTables().size()); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, TablesManager.GetSchemaPresets().size()); @@ -441,9 +443,11 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP NTabletFlatExecutor::TTransactionContext& txc) { NIceDb::TNiceDb db(txc.DB); - const auto pathId = TInternalPathId::FromRawValue(alterProto.GetPathId()); - Y_ABORT_UNLESS(TablesManager.HasTable(pathId), "AlterTable on a dropped or non-existent table"); - + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(alterProto); + const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId); + AFL_VERIFY(internalPathId); + Y_ABORT_UNLESS(TablesManager.HasTable(*internalPathId), "AlterTable on a dropped or non-existent table"); + const auto& pathId = TUnifiedPathId(*internalPathId, schemeShardLocalPathId); LOG_S_DEBUG("AlterTable for pathId: " << pathId << " schema: " << alterProto.GetSchema() << " ttl settings: " << alterProto.GetTtlSettings() @@ -466,25 +470,28 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP if (ttlSettings.HasEnabled()) { usedTiers = NOlap::TTiering::GetUsedTiers(ttlSettings.GetEnabled()); } - ActivateTiering(pathId, usedTiers); + ActivateTiering(*internalPathId, usedTiers); } tableVerProto.SetSchemaPresetVersionAdj(alterProto.GetSchemaPresetVersionAdj()); - TablesManager.AddTableVersion(pathId, version, tableVerProto, schema, db); + TablesManager.AddTableVersion(*internalPathId, version, tableVerProto, schema, db); } void TColumnShard::RunDropTable(const NKikimrTxColumnShard::TDropTable& dropProto, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) { NIceDb::TNiceDb db(txc.DB); - const auto pathId = TInternalPathId::FromRawValue(dropProto.GetPathId()); - if (!TablesManager.HasTable(pathId)) { + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(dropProto); + const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId); + AFL_VERIFY(internalPathId); + const auto& pathId = TUnifiedPathId(*internalPathId, schemeShardLocalPathId); + if (!TablesManager.HasTable(*internalPathId)) { LOG_S_DEBUG("DropTable for unknown or deleted pathId: " << pathId << " at tablet " << TabletID()); return; } LOG_S_DEBUG("DropTable for pathId: " << pathId << " at tablet " << TabletID()); - TablesManager.DropTable(pathId, version, db); + TablesManager.DropTable(*internalPathId, version, db); } void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto, const NOlap::TSnapshot& version, @@ -1287,7 +1294,7 @@ void TColumnShard::Handle(NOlap::NDataSharing::NEvents::TEvSendDataFromSource::T for (auto&& i : ev->Get()->Record.GetPathIdData()) { auto data = NOlap::NDataSharing::NEvents::TPathIdData::BuildFromProto(i, TablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex(), dsGroupSelector); AFL_VERIFY(data.IsSuccess())("error", data.GetErrorMessage()); - AFL_VERIFY(dataByPathId.emplace(TInternalPathId::FromRawValue(i.GetPathId()), data.DetachResult()).second); + AFL_VERIFY(dataByPathId.emplace(TInternalPathId::FromProto(i), data.DetachResult()).second); } const auto& schemeHistoryProto = ev->Get()->Record.GetSchemeHistory(); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index bc0d7eaf3567..6dba64d8f7e2 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -186,9 +186,10 @@ struct Schema : NIceDb::Schema { struct DropStep : Column<2, NScheme::NTypeIds::Uint64> {}; struct DropTxId : Column<3, NScheme::NTypeIds::Uint64> {}; struct TieringUsage: Column<4, NScheme::NTypeIds::String> {}; + struct SchemeShardLocalPathId : Column<5, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct TableVersionInfo : Table<(ui32)ECommonTables::TableVersionInfo> { @@ -787,6 +788,11 @@ struct Schema : NIceDb::Schema { db.Table().Key(pathId.GetRawValue()).Update(); } + static void SaveTableSchemeShardLocalPathId(NIceDb::TNiceDb& db, const TInternalPathId pathId, const TSchemeShardLocalPathId schemeShardLocalPathId) { + db.Table().Key(pathId.GetRawValue()).Update( + NIceDb::TUpdate(schemeShardLocalPathId.GetRawValue()) + ); + } static void SaveTableVersionInfo( NIceDb::TNiceDb& db, diff --git a/ydb/core/tx/columnshard/common/path_id.cpp b/ydb/core/tx/columnshard/common/path_id.cpp index 9ab28e86ab1b..717fbefad7a9 100644 --- a/ydb/core/tx/columnshard/common/path_id.cpp +++ b/ydb/core/tx/columnshard/common/path_id.cpp @@ -1,6 +1,189 @@ #include "path_id.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NKikimr::NColumnShard { + +//Explicit specializations for protos that hold TInternalPathId + +template<> +TInternalPathId TInternalPathId::FromProto(const NKikimrTxColumnShard::TTableVersionInfo& proto) { + return TInternalPathId(proto.GetPathId()); +} +template<> +void TInternalPathId::ToProto(NKikimrTxColumnShard::TTableVersionInfo& proto) const { + proto.SetPathId(PathId); +} + +template<> +TInternalPathId TInternalPathId::FromProto(const NKikimrTxColumnShard::TInternalOperationData& proto) { + return TInternalPathId(proto.GetPathId()); +} +template<> +void TInternalPathId::ToProto(NKikimrTxColumnShard::TInternalOperationData& proto) const { + proto.SetPathId(PathId); +} + +//TODO revise me when implementing reshading +template<> +TInternalPathId TInternalPathId::FromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto) { + return TInternalPathId(proto.GetPathId()); +} +template<> +void TInternalPathId::ToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const { + proto.SetPathId(PathId); +} + +//TODO revise me when implementing reshading +template<> +TInternalPathId TInternalPathId::FromProto(const NKikimrColumnShardDataSharingProto::TPathIdData& proto) { + return TInternalPathId(proto.GetPathId()); +} +template<> +void TInternalPathId::ToProto(NKikimrColumnShardDataSharingProto::TPathIdData& proto) const { + proto.SetPathId(PathId); +} + +//TODO revise me when implementing backup +template<> +TInternalPathId TInternalPathId::FromProto(const NKikimrColumnShardExportProto::TIdentifier& proto) { + return TInternalPathId(proto.GetPathId()); +} +template<> +void TInternalPathId::ToProto(NKikimrColumnShardExportProto::TIdentifier& proto) const { + proto.SetPathId(PathId); +} + + +//Explicit specialization for protos that hold SchemeShardLocalPathId + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TEvWrite& proto) { + return TSchemeShardLocalPathId(proto.GetTableId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TEvWrite& proto) const { + proto.SetTableId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TEvWriteResult& proto) { + return TSchemeShardLocalPathId(proto.GetTableId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TEvWriteResult& proto) const { + proto.SetTableId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TEvRead& proto) { + return TSchemeShardLocalPathId(proto.GetTableId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TEvRead& proto) const { + proto.SetTableId(PathId); +} +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TEvReadResult& proto) { + return TSchemeShardLocalPathId(proto.GetTableId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TEvReadResult& proto) const { + proto.SetTableId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TCreateTable& proto) { + return TSchemeShardLocalPathId(proto.GetPathId()); +} + +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TCreateTable& proto) const { + proto.SetPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TAlterTable& proto) { + return TSchemeShardLocalPathId(proto.GetPathId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TAlterTable& proto) const { + proto.SetPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxColumnShard::TDropTable& proto) { + return TSchemeShardLocalPathId(proto.GetPathId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxColumnShard::TDropTable& proto) const { + proto.SetPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrSchemeOp::TGranuleShardingInfo& proto) { + return TSchemeShardLocalPathId(proto.GetPathId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrSchemeOp::TGranuleShardingInfo& proto) const { + proto.SetPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrTxDataShard::TEvKqpScan& proto) { + return TSchemeShardLocalPathId(proto.GetLocalPathId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrTxDataShard::TEvKqpScan& proto) const { + proto.SetLocalPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrDataEvents::TTableId& proto) { + return TSchemeShardLocalPathId(proto.GetTableId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrDataEvents::TTableId& proto) const { + proto.SetTableId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrDataEvents::TLock& proto) { + return TSchemeShardLocalPathId(proto.GetPathId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrDataEvents::TLock& proto) const { + proto.SetPathId(PathId); +} + +template<> +TSchemeShardLocalPathId TSchemeShardLocalPathId::FromProto(const NKikimrProto::TPathID& proto) { + return TSchemeShardLocalPathId(proto.GetLocalId()); +} +template<> +void TSchemeShardLocalPathId::ToProto(NKikimrProto::TPathID& proto) const { + proto.SetLocalId(PathId); +} +} //namespace NKikimr::NColumnShard + +template<> +void Out(IOutputStream& s, const NKikimr::NColumnShard::TInternalPathId& v) { + s << v.GetRawValue(); +} + +template<> +void Out(IOutputStream& s, const NKikimr::NColumnShard::TSchemeShardLocalPathId& v) { + s << v.GetRawValue(); +} template<> -void Out(IOutputStream& s, TTypeTraits::TFuncParam v) { - s << v.GetRawValue(); +void Out(IOutputStream& s, const NKikimr::NColumnShard::TUnifiedPathId& v) { + s << "{internal: " << v.InternalPathId << ", ss: " << v.SchemeShardLocalPathId << "}"; } diff --git a/ydb/core/tx/columnshard/common/path_id.h b/ydb/core/tx/columnshard/common/path_id.h index b5c18912de10..207ffae747b2 100644 --- a/ydb/core/tx/columnshard/common/path_id.h +++ b/ydb/core/tx/columnshard/common/path_id.h @@ -20,22 +20,78 @@ class TInternalPathId { TInternalPathId& operator=(const TInternalPathId&) = default; TInternalPathId& operator=(TInternalPathId&&) = default; + explicit operator bool() const { + return PathId != 0; + } + static TInternalPathId FromRawValue(const ui64 pathId) { return TInternalPathId(pathId); } + ui64 GetRawValue() const { + return PathId; + } + + //Template function whithout generic implementation + //Must be explicitly instantiated for messages that hold internal path id + template + static TInternalPathId FromProto(const Proto& proto); + + //Template function whithout generic implementation + //Must be explicitly instantiated for messages that hold internal path id + template + void ToProto(Proto& proto) const; + auto operator<=>(const TInternalPathId&) const = default; +}; + +static_assert(sizeof(TInternalPathId) == sizeof(ui64)); + +class TSchemeShardLocalPathId { + ui64 PathId; +private: + explicit TSchemeShardLocalPathId(const ui64 pathId) + : PathId(pathId) { + } +public: + TSchemeShardLocalPathId() + : PathId(0) { + } explicit operator bool() const { return PathId != 0; } + static TSchemeShardLocalPathId FromRawValue(const ui64 pathId) { + return TSchemeShardLocalPathId(pathId); + } + ui64 GetRawValue() const { return PathId; } - auto operator<=>(const TInternalPathId&) const = default; + //Templated function whithout generic implementation + //Must be explicitly instantiated for messages that hold SchemeShardLocalPathId + template + static TSchemeShardLocalPathId FromProto(const Proto& proto); + + //Templated function whithout generic implementation + //Must be explicitly instantiated for messages that hold SchemeShardLocalPathId + template + void ToProto(Proto& proto) const; + + auto operator<=>(const TSchemeShardLocalPathId&) const = default; +}; + +static_assert(sizeof(TSchemeShardLocalPathId) == sizeof(ui64)); + +struct TUnifiedPathId { + TInternalPathId InternalPathId; + TSchemeShardLocalPathId SchemeShardLocalPathId; + explicit operator bool() const { + return InternalPathId && SchemeShardLocalPathId; + } }; -static_assert(sizeof(TInternalPathId)==sizeof(ui64)); +static_assert(sizeof(TUnifiedPathId) == 2 * sizeof(ui64)); } //namespace NKikimr::NColumnShard @@ -43,6 +99,12 @@ namespace NKikimr::NOlap { using TInternalPathId = NColumnShard::TInternalPathId; +struct IPathIdTranslator { + virtual ~IPathIdTranslator() {} + virtual std::optional ResolveSchemeShardLocalPathId(const TInternalPathId internalPathId) const = 0; + virtual std::optional ResolveInternalPathId(const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId) const = 0; +}; + } //namespace NKikimr::NOlap template <> @@ -51,3 +113,10 @@ struct THash { return THash()(p.GetRawValue()); } }; + +template <> +struct THash { + size_t operator()(const NKikimr::NColumnShard::TSchemeShardLocalPathId& p) const { + return THash()(p.GetRawValue()); + } +}; diff --git a/ydb/core/tx/columnshard/common/ya.make b/ydb/core/tx/columnshard/common/ya.make index ec817047696c..86c71afb41ba 100644 --- a/ydb/core/tx/columnshard/common/ya.make +++ b/ydb/core/tx/columnshard/common/ya.make @@ -17,6 +17,10 @@ PEERDIR( contrib/libs/apache/arrow ydb/core/formats/arrow ydb/core/tx/columnshard/common/protos + ydb/core/tx/columnshard/data_sharing/protos + ydb/core/tx/columnshard/transactions/protos + ydb/core/tx/columnshard/export/protos + ydb/core/scheme/protos ) GENERATE_ENUM_SERIALIZATION(portion.h) diff --git a/ydb/core/tx/columnshard/counters/column_tables.cpp b/ydb/core/tx/columnshard/counters/column_tables.cpp index bbf68cf8836f..47210d512942 100644 --- a/ydb/core/tx/columnshard/counters/column_tables.cpp +++ b/ydb/core/tx/columnshard/counters/column_tables.cpp @@ -3,7 +3,7 @@ namespace NKikimr::NColumnShard { -std::shared_ptr TColumnTablesCounters::GetPathIdCounter(TInternalPathId pathId) { +std::shared_ptr TColumnTablesCounters::GetPathIdCounter(const TInternalPathId pathId) { auto findCounter = PathIdCounters.FindPtr(pathId); if (findCounter) { return *findCounter; diff --git a/ydb/core/tx/columnshard/counters/column_tables.h b/ydb/core/tx/columnshard/counters/column_tables.h index 29cfcb4b860a..58f5a12f42c3 100644 --- a/ydb/core/tx/columnshard/counters/column_tables.h +++ b/ydb/core/tx/columnshard/counters/column_tables.h @@ -31,7 +31,7 @@ class TColumnTablesCounters { output.SetLastUpdateTime(LastUpdateTime->MilliSeconds()); } - std::shared_ptr GetPathIdCounter(TInternalPathId pathId); + std::shared_ptr GetPathIdCounter(const TInternalPathId pathId); }; class TSingleColumnTableCounters { diff --git a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h index 3d8e9b93e263..931bcf354ae0 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h +++ b/ydb/core/tx/columnshard/data_sharing/destination/events/transfer.h @@ -28,7 +28,7 @@ class TPathIdData { if (!proto.HasPathId()) { return TConclusionStatus::Fail("no path id in proto"); } - PathId = TInternalPathId::FromRawValue(proto.GetPathId()); + PathId = TInternalPathId::FromProto(proto); for (auto&& portionProto : proto.GetPortions()) { const auto schema = versionedIndex.GetSchemaVerified(portionProto.GetSchemaVersion()); TConclusion portion = TPortionDataAccessor::BuildFromProto(portionProto, schema->GetIndexInfo(), groupSelector); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 8352294b9492..271d1a330158 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -207,10 +207,6 @@ class TColumnEngineForLogs: public IColumnEngine { return GranulesStorage->GetGranuleOptional(pathId); } - std::vector> GetTables(const std::optional pathIdFrom, const std::optional pathIdTo) const { - return GranulesStorage->GetTables(pathIdFrom, pathIdTo); - } - ui64 GetTabletId() const { return TabletId; } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 809adabf12ef..dff837746e95 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -53,7 +53,7 @@ ui64 TPortionInfo::GetApproxChunksCount(const ui32 schemaColumnsCount) const { } void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const { - proto.SetPathId(PathId.GetRawValue()); + PathId.ToProto(proto); proto.SetPortionId(PortionId); proto.SetSchemaVersion(GetSchemaVersionVerified()); if (!RemoveSnapshot.IsZero()) { @@ -64,7 +64,7 @@ void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortion } TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto) { - PathId = TInternalPathId::FromRawValue(proto.GetPathId()); + PathId = TInternalPathId::FromProto(proto); PortionId = proto.GetPortionId(); SchemaVersion = proto.GetSchemaVersion(); if (!SchemaVersion) { diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp index 803aa7030543..58896e6d529e 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp @@ -14,7 +14,8 @@ std::shared_ptr TDataStorageAccessor::Select(const TReadDescription return std::make_shared(); } AFL_VERIFY(readDescription.PKRangesFilter); - return Index->Select(readDescription.PathId, readDescription.GetSnapshot(), *readDescription.PKRangesFilter, withUncommitted); + return Index->Select(readDescription.PathId.InternalPathId, readDescription.GetSnapshot(), + *readDescription.PKRangesFilter, withUncommitted); } ISnapshotSchema::TPtr TReadMetadataBase::GetLoadSchemaVerified(const TPortionInfo& portion) const { @@ -26,7 +27,8 @@ ISnapshotSchema::TPtr TReadMetadataBase::GetLoadSchemaVerified(const TPortionInf std::vector TDataStorageAccessor::GetCommitedBlobs(const TReadDescription& readDescription, const std::shared_ptr& pkSchema, const std::optional lockId, const TSnapshot& reqSnapshot) const { AFL_VERIFY(readDescription.PKRangesFilter); - return std::move(InsertTable->Read(readDescription.PathId, lockId, reqSnapshot, pkSchema, &*readDescription.PKRangesFilter)); + return std::move(InsertTable->Read(readDescription.PathId.InternalPathId, lockId, reqSnapshot, + pkSchema, &*readDescription.PKRangesFilter)); } } // namespace NKikimr::NOlap::NReader diff --git a/ydb/core/tx/columnshard/engines/reader/common/description.h b/ydb/core/tx/columnshard/engines/reader/common/description.h index 8c1edddea79d..2650a051addb 100644 --- a/ydb/core/tx/columnshard/engines/reader/common/description.h +++ b/ydb/core/tx/columnshard/engines/reader/common/description.h @@ -32,7 +32,7 @@ struct TReadDescription { // Table ui64 TxId = 0; std::optional LockId; - TInternalPathId PathId; + NColumnShard::TUnifiedPathId PathId; TString TableName; bool ReadNothing = false; // Less[OrEqual], Greater[OrEqual] or both diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp index 934a603e4df3..89a07f35b282 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NOlap::NReader::NCommon { TConclusionStatus TReadMetadata::Init( const NColumnShard::TColumnShard* owner, const TReadDescription& readDescription, const TDataStorageAccessor& dataAccessor) { SetPKRangesFilter(readDescription.PKRangesFilter); - InitShardingInfo(readDescription.PathId); + InitShardingInfo(readDescription.PathId.InternalPathId); TxId = readDescription.TxId; LockId = readDescription.LockId; if (LockId) { @@ -88,7 +88,7 @@ void TReadMetadata::DoOnReadFinished(NColumnShard::TColumnShard& owner) const { for (auto&& i : GetConflictableLockIds()) { conflicts.Add(i, lock); } - auto writer = std::make_shared(PathId, conflicts); + auto writer = std::make_shared(PathId.InternalPathId, conflicts); owner.GetOperationsManager().AddEventForLock(owner, lock, writer); } } @@ -98,7 +98,7 @@ void TReadMetadata::DoOnBeforeStartReading(NColumnShard::TColumnShard& owner) co return; } auto evWriter = std::make_shared( - PathId, GetResultSchema()->GetIndexInfo().GetPrimaryKey(), GetPKRangesFilterPtr(), GetConflictableLockIds()); + PathId.InternalPathId, GetResultSchema()->GetIndexInfo().GetPrimaryKey(), GetPKRangesFilterPtr(), GetConflictableLockIds()); owner.GetOperationsManager().AddEventForLock(owner, *LockId, evWriter); } @@ -109,7 +109,7 @@ void TReadMetadata::DoOnReplyConstruction(const ui64 tabletId, NKqp::NInternalIm lockInfo.SetGeneration(LockSharingInfo->GetGeneration()); lockInfo.SetDataShard(tabletId); lockInfo.SetCounter(LockSharingInfo->GetCounter()); - lockInfo.SetPathId(PathId.GetRawValue()); + PathId.SchemeShardLocalPathId.ToProto(lockInfo); lockInfo.SetHasWrites(LockSharingInfo->HasWrites()); if (LockSharingInfo->IsBroken()) { scanData.LocksInfo.BrokenLocks.emplace_back(std::move(lockInfo)); diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h index 969f2e5b24b9..d13b926fe5ba 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.h @@ -17,7 +17,7 @@ class TReadMetadata: public TReadMetadataBase { using TBase = TReadMetadataBase; private: - const TInternalPathId PathId; + const NColumnShard::TUnifiedPathId PathId; std::shared_ptr BrokenWithCommitted = std::make_shared(); std::shared_ptr LockSharingInfo; @@ -110,7 +110,7 @@ class TReadMetadata: public TReadMetadataBase { return GetProgram().HasProcessingColumnIds(); } - TInternalPathId GetPathId() const { + NColumnShard::TUnifiedPathId GetPathId() const { return PathId; } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp index e0c8e955afab..c189da40bffc 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.cpp @@ -18,12 +18,12 @@ NKikimr::TConclusionStatus TMetadataFromStore::DoFillMetadata(const NColumnShard THashSet pathIds; AFL_VERIFY(read.PKRangesFilter); for (auto&& filter : *read.PKRangesFilter) { - const auto fromPathId = TInternalPathId::FromRawValue(*filter.GetPredicateFrom().Get(0, 0, 1)); - const auto toPathId = TInternalPathId::FromRawValue(*filter.GetPredicateTo().Get(0, 0, Max())); - auto pathInfos = logsIndex->GetTables(fromPathId, toPathId); - for (auto&& pathInfo : pathInfos) { - if (pathIds.emplace(pathInfo->GetPathId()).second) { - metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted(), metadata->GetRequestSnapshot())); + const auto fromPathId = NColumnShard::TSchemeShardLocalPathId::FromRawValue(*filter.GetPredicateFrom().Get(0, 0, 1)); + const auto toPathId = NColumnShard::TSchemeShardLocalPathId::FromRawValue(*filter.GetPredicateTo().Get(0, 0, Max())); + for (const auto& [schemeShardLocalPathId, internalPathId]: shard->GetTablesManager().ResolveInternalPathIds(fromPathId, toPathId)) { + const auto& pathInfo = logsIndex->GetGranulePtrVerified(internalPathId); + if (pathIds.emplace(internalPathId).second) { + metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, schemeShardLocalPathId, metadata->IsDescSorted(), metadata->GetRequestSnapshot())); } } } @@ -44,15 +44,16 @@ NKikimr::TConclusionStatus TMetadataFromTable::DoFillMetadata(const NColumnShard return TConclusionStatus::Success(); } AFL_VERIFY(read.PKRangesFilter); + const auto& schemeShardLocalPathId = read.PathId.SchemeShardLocalPathId; for (auto&& filter : *read.PKRangesFilter) { - const auto fromPathId = TInternalPathId::FromRawValue(*filter.GetPredicateFrom().Get(0, 0, 1)); - const auto toPathId = TInternalPathId::FromRawValue(*filter.GetPredicateTo().Get(0, 0, Max())); - if (fromPathId <= read.PathId && read.PathId <= toPathId) { - auto pathInfo = logsIndex->GetGranuleOptional(read.PathId); + const auto fromPathId = NColumnShard::TSchemeShardLocalPathId::FromRawValue(*filter.GetPredicateFrom().Get(0, 0, 1)); + const auto toPathId = NColumnShard::TSchemeShardLocalPathId::FromRawValue(*filter.GetPredicateTo().Get(0, 0, Max())); + if ((fromPathId <= schemeShardLocalPathId) && (schemeShardLocalPathId <= toPathId)) { + auto pathInfo = logsIndex->GetGranuleOptional(read.PathId.InternalPathId); if (!pathInfo) { continue; } - metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, metadata->IsDescSorted(), metadata->GetRequestSnapshot())); + metadata->IndexGranules.emplace_back(BuildGranuleView(*pathInfo, schemeShardLocalPathId, metadata->IsDescSorted(), metadata->GetRequestSnapshot())); break; } } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h index 8caecc01f5b7..46c027275dee 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/filler.h @@ -9,8 +9,8 @@ class IMetadataFiller { private: virtual TConclusionStatus DoFillMetadata(const NColumnShard::TColumnShard* shard, const std::shared_ptr& metadata, const TReadDescription& read) const = 0; - virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const { - return NAbstract::TGranuleMetaView(granule, reverse, reqSnapshot); + virtual NAbstract::TGranuleMetaView DoBuildGranuleView(const TGranuleMeta& granule, const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId, const bool reverse, const TSnapshot& reqSnapshot) const { + return NAbstract::TGranuleMetaView(granule, schemeShardLocalPathId, reverse, reqSnapshot); } public: virtual ~IMetadataFiller() = default; @@ -19,8 +19,8 @@ class IMetadataFiller { return DoFillMetadata(shard, metadata, read); } - NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) const { - return DoBuildGranuleView(granule, reverse, reqSnapshot); + NAbstract::TGranuleMetaView BuildGranuleView(const TGranuleMeta& granule, const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId, const bool reverse, const TSnapshot& reqSnapshot) const { + return DoBuildGranuleView(granule, schemeShardLocalPathId, reverse, reqSnapshot); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h index cb3e2b370ac7..a2aed67cf65e 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/abstract/granule_view.h @@ -8,12 +8,12 @@ namespace NKikimr::NOlap::NReader::NSysView::NAbstract { class TGranuleMetaView { private: using TPortions = std::deque>; - YDB_READONLY_DEF(TInternalPathId, PathId); + YDB_READONLY_DEF(NColumnShard::TUnifiedPathId, PathId); YDB_READONLY_DEF(TPortions, Portions); YDB_READONLY_DEF(std::vector, OptimizerTasks); public: - TGranuleMetaView(const TGranuleMeta& granule, const bool reverse, const TSnapshot& reqSnapshot) - : PathId(granule.GetPathId()) + TGranuleMetaView(const TGranuleMeta& granule, const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId, const bool reverse, const TSnapshot& reqSnapshot) + : PathId({granule.GetPathId(), schemeShardLocalPathId}) { for (auto&& i : granule.GetPortions()) { if (i.second->IsRemovedFor(reqSnapshot)) { @@ -41,7 +41,7 @@ class TGranuleMetaView { } bool operator<(const TGranuleMetaView& item) const { - return PathId < item.PathId; + return PathId.SchemeShardLocalPathId < item.PathId.SchemeShardLocalPathId; } std::shared_ptr PopFrontPortion() { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 9ad6dd03c183..961397d9d363 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NOlap::NReader::NSysView::NChunks { void TStatsIterator::AppendStats( - const std::vector>& builders, const TPortionDataAccessor& portionPtr) const { + const std::vector>& builders, const NColumnShard::TSchemeShardLocalPathId schemshardLocalPathId, const TPortionDataAccessor& portionPtr) const { const TPortionInfo& portion = portionPtr.GetPortionInfo(); auto portionSchema = ReadMetadata->GetLoadSchemaVerified(portion); auto it = PortionType.find(portion.GetProduced()); @@ -35,7 +35,7 @@ void TStatsIterator::AppendStats( arrow::util::string_view lastColumnName; arrow::util::string_view lastTierName; for (auto&& r : records) { - NArrow::Append(*builders[0], portion.GetPathId().GetRawValue()); + NArrow::Append(*builders[0], schemshardLocalPathId.GetRawValue()); NArrow::Append(*builders[1], prodView); NArrow::Append(*builders[2], ReadMetadata->GetTabletId()); NArrow::Append(*builders[3], r->GetMeta().GetRecordsCount()); @@ -92,7 +92,7 @@ void TStatsIterator::AppendStats( std::reverse(indexes.begin(), indexes.end()); } for (auto&& r : indexes) { - NArrow::Append(*builders[0], portion.GetPathId().GetRawValue()); + NArrow::Append(*builders[0], schemshardLocalPathId.GetRawValue()); NArrow::Append(*builders[1], prodView); NArrow::Append(*builders[2], ReadMetadata->GetTabletId()); NArrow::Append(*builders[3], r->GetRecordsCount()); @@ -143,7 +143,7 @@ bool TStatsIterator::AppendStats(const std::vectorsecond.GetRecordsVerified().size() + it->second.GetIndexesVerified().size(); - AppendStats(builders, it->second); + AppendStats(builders, granule.GetPathId().SchemeShardLocalPathId, it->second); granule.PopFrontPortion(); FetchedAccessors.erase(it); if (recordsCount > 10000) { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h index 70381bab0514..828cabbe8d64 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h @@ -68,7 +68,7 @@ class TStatsIterator: public NAbstract::TStatsIterator>& builders, NAbstract::TGranuleMetaView& granule) const override; virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override; - void AppendStats(const std::vector>& builders, const TPortionDataAccessor& portion) const; + void AppendStats(const std::vector>& builders, const NColumnShard::TSchemeShardLocalPathId schemshardLocalPathId, const TPortionDataAccessor& portion) const; class TApplyResult: public IApplyAction { private: diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp index b714821d7c9f..0a58782671bf 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NOlap::NReader::NSysView::NGranules { bool TStatsIterator::AppendStats(const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const { - NArrow::Append(*builders[0], granule.GetPathId().GetRawValue()); + NArrow::Append(*builders[0], granule.GetPathId().SchemeShardLocalPathId.GetRawValue()); NArrow::Append(*builders[1], ReadMetadata->GetTabletId()); NArrow::Append(*builders[2], granule.GetPortions().size()); NArrow::Append(*builders[3], HostNameField); diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp index c43e82b690fd..6f6b0a61ae03 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NOlap::NReader::NSysView::NOptimizer { bool TStatsIterator::AppendStats(const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const { for (auto&& i : granule.GetOptimizerTasks()) { - NArrow::Append(*builders[0], granule.GetPathId().GetRawValue()); + NArrow::Append(*builders[0], granule.GetPathId().SchemeShardLocalPathId.GetRawValue()); NArrow::Append(*builders[1], ReadMetadata->GetTabletId()); NArrow::Append(*builders[2], i.GetTaskId()); NArrow::Append(*builders[3], HostNameField); diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h index d31545fe619c..cf722775552a 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/optimizer/optimizer.h @@ -41,8 +41,9 @@ class TStatsIterator : public NAbstract::TStatsIterator>& builders, const TPortionInfo& portion) const { - NArrow::Append(*builders[0], portion.GetPathId().GetRawValue()); +void TStatsIterator::AppendStats(const std::vector>& builders, const TPortionInfo& portion, const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId) const { + NArrow::Append(*builders[0], schemeShardLocalPathId.GetRawValue()); const std::string prod = ::ToString(portion.GetProduced()); NArrow::Append(*builders[1], prod); NArrow::Append(*builders[2], ReadMetadata->GetTabletId()); @@ -45,7 +45,8 @@ bool TStatsIterator::AppendStats(const std::vectorGetPathId() == granule.GetPathId().InternalPathId); + AppendStats(builders, *portion, granule.GetPathId().SchemeShardLocalPathId); if (recordsCount >= 10000) { break; } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.h b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.h index 9f5fd67fb8c9..4bded2494dd0 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/portions/portions.h @@ -31,7 +31,7 @@ class TStatsIterator : public NAbstract::TStatsIterator; virtual bool AppendStats(const std::vector>& builders, NAbstract::TGranuleMetaView& granule) const override; virtual ui32 PredictRecordsCount(const NAbstract::TGranuleMetaView& granule) const override; - void AppendStats(const std::vector>& builders, const TPortionInfo& portion) const; + void AppendStats(const std::vector>& builders, const TPortionInfo& portion, const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId) const; public: using TBase::TBase; }; diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp index 863407ef3262..7a0a46ff0a30 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_internal_scan.cpp @@ -48,7 +48,7 @@ void TTxInternalScan::Complete(const TActorContext& ctx) { read.SetScanIdentifier(request.TaskIdentifier); read.PathId = request.GetPathId(); read.LockId = LockId; - read.ReadNothing = !Self->TablesManager.HasTable(read.PathId); + read.ReadNothing = !Self->TablesManager.HasTable(read.PathId.InternalPathId); read.DeduplicationPolicy = EDeduplicationPolicy::PREVENT_DUPLICATES; std::unique_ptr scannerConstructor(new NPlain::TIndexScannerConstructor(context)); read.ColumnIds = request.GetColumnIds(); diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index 9c2865b556ff..478d892ad556 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -71,8 +71,11 @@ void TTxScan::Complete(const TActorContext& ctx) { if (request.HasLockTxId()) { read.LockId = request.GetLockTxId(); } - read.PathId = TInternalPathId::FromRawValue(request.GetLocalPathId()); - read.ReadNothing = !Self->TablesManager.HasTable(read.PathId); + + const auto& schemeShardLocalPathId = NColumnShard::TSchemeShardLocalPathId::FromProto(request); + const auto& internalPathId = Self->TablesManager.ResolveInternalPathId(schemeShardLocalPathId); + read.PathId = NColumnShard::TUnifiedPathId{internalPathId ? *internalPathId : TInternalPathId{}, schemeShardLocalPathId}; + read.ReadNothing = !Self->TablesManager.HasTable(read.PathId.InternalPathId); read.TableName = table; const TString defaultReader = diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.h b/ydb/core/tx/columnshard/engines/storage/granule/storage.h index b22f9f0c089e..b38b081f31b1 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.h @@ -170,19 +170,6 @@ class TGranulesStorage { } } - std::vector> GetTables(const std::optional pathIdFrom, const std::optional pathIdTo) const { - std::vector> result; - for (auto&& i : Tables) { - if (pathIdFrom && i.first < *pathIdFrom) { - continue; - } - if (pathIdTo && i.first > *pathIdTo) { - continue; - } - result.emplace_back(i.second); - } - return result; - } std::shared_ptr GetPortionOptional(const TInternalPathId pathId, const ui64 portionId) const { auto it = Tables.find(pathId); diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp index b6b5ff586b47..503a64e1c10f 100644 --- a/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp +++ b/ydb/core/tx/columnshard/engines/writer/buffer/actor2.cpp @@ -49,7 +49,7 @@ void TActor::Handle(TEvAddInsertedDataToBuffer::TPtr& ev) { AFL_VERIFY(evBase->GetWriteData()->GetBlobsAction()->GetStorageId() == NOlap::IStoragesManager::DefaultStorageId); SumSize += evBase->GetWriteData()->GetSize(); - const TInternalPathId pathId = evBase->GetWriteData()->GetWriteMeta().GetTableId(); + const auto& pathId = evBase->GetWriteData()->GetWriteMeta().GetPathId().InternalPathId; const ui64 schemaVersion = evBase->GetContext()->GetActualSchema()->GetVersion(); TAggregationId aggrId(pathId, schemaVersion, evBase->GetWriteData()->GetWriteMeta().GetModificationType()); auto it = Aggregations.find(aggrId); diff --git a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp index 8176ad769cd9..f6ebdc8b1426 100644 --- a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp +++ b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp @@ -50,7 +50,7 @@ std::shared_ptr TWideSerializedBatch::BuildInsertionU auto schemeVersion = GetAggregation().GetSchemaVersion(); auto tableSchema = owner.GetTablesManager().GetPrimaryIndex()->GetVersionedIndex().GetSchemaVerified(schemeVersion); - return std::make_shared(writeMeta.GetTableId(), blobRange, meta, tableSchema->GetVersion(), SplittedBlobs.GetData()); + return std::make_shared(writeMeta.GetPathId().InternalPathId, blobRange, meta, tableSchema->GetVersion(), SplittedBlobs.GetData()); } void TWritingBuffer::InitReadyInstant(const TMonotonic /*instant*/) { diff --git a/ydb/core/tx/columnshard/export/common/identifier.cpp b/ydb/core/tx/columnshard/export/common/identifier.cpp index 1f6de289128b..f9c72466a0b6 100644 --- a/ydb/core/tx/columnshard/export/common/identifier.cpp +++ b/ydb/core/tx/columnshard/export/common/identifier.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NOlap::NExport { NKikimr::TConclusionStatus TIdentifier::DeserializeFromProto(const NKikimrColumnShardExportProto::TIdentifier& proto) { - PathId = TInternalPathId::FromRawValue(proto.GetPathId()); + PathId = TInternalPathId::FromProto(proto); if (!PathId) { return TConclusionStatus::Fail("Incorrect pathId (zero)"); } @@ -33,7 +33,7 @@ NKikimr::TConclusion TIdentifier::BuildFro NKikimrColumnShardExportProto::TIdentifier TIdentifier::SerializeToProto() const { NKikimrColumnShardExportProto::TIdentifier result; - result.SetPathId(PathId.GetRawValue()); + PathId.ToProto(result); return result; } diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index e94b510f6982..52cd8b3fee2b 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -356,6 +357,16 @@ class ICSController { virtual THashMap> GetExternalDataLocks() const { return {}; } + + virtual ui64 GetInternalPathIdOffset(const ui64 /*tabletId*/) const { + return 0; + } + + virtual void OnAddPathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { + } + virtual void OnDeletePathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { + } + }; class TControllers { diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index c6d3be3089a3..e4b720d2c71c 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -92,23 +92,6 @@ void TController::DoOnTabletStopped(const ::NKikimr::NColumnShard::TColumnShard& AFL_VERIFY(ShardActuals.erase(shard.TabletID())); } -std::vector TController::GetPathIds(const ui64 tabletId) const { - TGuard g(Mutex); - std::vector result; - for (auto&& i : ShardActuals) { - if (i.first == tabletId) { - const auto& index = i.second->GetIndexAs(); - std::vector> granules = index.GetTables({}, {}); - - for (auto&& g : granules) { - result.emplace_back(g->GetPathId()); - } - break; - } - } - return result; -} - bool TController::IsTrivialLinks() const { TGuard g(Mutex); for (auto&& i : ShardActuals) { diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index e389c9c4c63a..a55a1447ecf5 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -1,4 +1,5 @@ #pragma once +#include #include "ro_controller.h" #include @@ -47,7 +48,55 @@ class TController: public TReadOnlyController { std::set DisabledBackgrounds; TMutex ActiveTabletsMutex; - std::set ActiveTablets; + + using TInternalPathId = NKikimr::NColumnShard::TInternalPathId; + using TSchemeShardLocalPathId = NKikimr::NColumnShard::TSchemeShardLocalPathId; + using TUnifiedPathId = NKikimr::NColumnShard::TUnifiedPathId; + + class TPathIdTranslator: public NOlap::IPathIdTranslator { + THashMap InternalToSchemeShardLocal; + THashMap SchemeShardLocalToInternal; + public: + void AddPathId(const TUnifiedPathId& pathId) { + AFL_VERIFY(InternalToSchemeShardLocal.emplace(pathId.InternalPathId, pathId.SchemeShardLocalPathId).second); + AFL_VERIFY(SchemeShardLocalToInternal.emplace(pathId.SchemeShardLocalPathId, pathId.InternalPathId).second); + } + void DeletePathId(const TUnifiedPathId& pathId) { + InternalToSchemeShardLocal.erase(pathId.InternalPathId); + SchemeShardLocalToInternal.erase(pathId.SchemeShardLocalPathId); + } + public: + THashSet GetInternalPathIds() const { + THashSet result; + for (const auto& [internalPathId, schemeShardLocalPathId]: InternalToSchemeShardLocal) { + result.emplace(internalPathId); + } + return result; + } + THashSet GetSchemeShardLocalPathIds() const { + THashSet result; + for (const auto& [internalPathId, schemeShardLocalPathId]: InternalToSchemeShardLocal) { + result.emplace(schemeShardLocalPathId); + } + return result; + } + + public: //NOlap::IPathIdTranslator + virtual std::optional ResolveSchemeShardLocalPathId(const TInternalPathId internalPathId) const override { + if (const auto* p = InternalToSchemeShardLocal.FindPtr(internalPathId)) { + return {*p}; + } + return std::nullopt; + } + virtual std::optional ResolveInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) const override { + if (const auto* p = SchemeShardLocalToInternal.FindPtr(schemeShardLocalPathId)) { + return {*p}; + } + return std::nullopt; + } + }; + THashMap> ActiveTablets; + THashMap> ExternalLocks; @@ -283,8 +332,6 @@ class TController: public TReadOnlyController { return result; } - std::vector GetPathIds(const ui64 tabletId) const; - void SetExpectedShardsCount(const ui32 value) { ExpectedShardsCount = value; } @@ -306,7 +353,7 @@ class TController: public TReadOnlyController { void OnSwitchToWork(const ui64 tabletId) override { TGuard g(ActiveTabletsMutex); - ActiveTablets.emplace(tabletId); + ActiveTablets.emplace(tabletId, std::make_shared()); } void OnCleanupActors(const ui64 tabletId) override { @@ -314,9 +361,9 @@ class TController: public TReadOnlyController { ActiveTablets.erase(tabletId); } - ui64 GetActiveTabletsCount() const { + THashMap> GetActiveTablets() const { TGuard g(ActiveTabletsMutex); - return ActiveTablets.size(); + return ActiveTablets; } bool IsActiveTablet(const ui64 tabletId) const { @@ -328,8 +375,31 @@ class TController: public TReadOnlyController { RestartOnLocalDbTxCommitted = std::move(txInfo); } + const std::shared_ptr GetPathIdTranslator(const ui64 tabletId) { + TGuard g(ActiveTabletsMutex); + const auto* tablet = ActiveTablets.FindPtr(tabletId); + if (!tablet) { + return nullptr; + } + return *tablet; + } + virtual void OnAfterLocalTxCommitted( const NActors::TActorContext& ctx, const ::NKikimr::NColumnShard::TColumnShard& shard, const TString& txInfo) override; + + + virtual void OnAddPathId(const ui64 tabletId, const TUnifiedPathId& pathId) override { + TGuard g(ActiveTabletsMutex); + auto* tablet = ActiveTablets.FindPtr(tabletId); + AFL_VERIFY(tablet); + (*tablet)->AddPathId(pathId); + } + virtual void OnDeletePathId(const ui64 tabletId, const TUnifiedPathId& pathId) override { + auto* tablet = ActiveTablets.FindPtr(tabletId); + AFL_VERIFY(tablet); + (*tablet)->DeletePathId(pathId); + } + }; } diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 8506f9249b6f..a13776983451 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -193,6 +193,10 @@ class TReadOnlyController: public ICSController { HeadersSkippingOnSelect.Inc(); } } + + virtual ui64 GetInternalPathIdOffset(const ui64 tabletId) const override { + return 1000000 + static_cast(tabletId) % 1000; + } }; } diff --git a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp index 049c6c100b2a..41945707eaf2 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp @@ -9,7 +9,8 @@ namespace NKikimr::NOlap { std::unique_ptr TModificationRestoreTask::DoBuildRequestInitiator() const { - auto request = std::make_unique(LocalPathId, Snapshot, WriteData.GetWriteMeta().GetLockIdOptional()); + const auto& writeMetaData = WriteData.GetWriteMeta(); + auto request = std::make_unique(writeMetaData.GetPathId(), Snapshot, writeMetaData.GetLockIdOptional()); request->TaskIdentifier = GetTaskId(); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_RESTORE)("event", "restore_start")( "count", IncomingData.HasContainer() ? IncomingData->num_rows() : 0)("task_id", WriteData.GetWriteMeta().GetId()); @@ -66,7 +67,6 @@ TModificationRestoreTask::TModificationRestoreTask(NEvWrite::TWriteData&& writeD writeData.GetWriteMeta().GetId() + "::" + ::ToString(writeData.GetWriteMeta().GetWriteId())) , WriteData(std::move(writeData)) , Merger(merger) - , LocalPathId(WriteData.GetWriteMeta().GetTableId()) , Snapshot(actualSnapshot) , IncomingData(incomingData) , Context(context) { diff --git a/ydb/core/tx/columnshard/operations/batch_builder/restore.h b/ydb/core/tx/columnshard/operations/batch_builder/restore.h index a545931b7255..9f601f3a3408 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/restore.h +++ b/ydb/core/tx/columnshard/operations/batch_builder/restore.h @@ -14,7 +14,6 @@ class TModificationRestoreTask: public NDataReader::IRestoreTask, public NColumn using TBase = NDataReader::IRestoreTask; NEvWrite::TWriteData WriteData; std::shared_ptr Merger; - const TInternalPathId LocalPathId; const TSnapshot Snapshot; NArrow::TContainerWithIndexes IncomingData; const TWritingContext Context; diff --git a/ydb/core/tx/columnshard/operations/events.h b/ydb/core/tx/columnshard/operations/events.h index bf0ac210c627..d91f6e84772f 100644 --- a/ydb/core/tx/columnshard/operations/events.h +++ b/ydb/core/tx/columnshard/operations/events.h @@ -100,9 +100,9 @@ class TInsertedPortions { i.GetWriteMeta().OnStage(NEvWrite::EWriteStage::Finished); AFL_VERIFY(!i.GetWriteMeta().HasLongTxId()); if (!pathId) { - pathId = i.GetWriteMeta().GetTableId(); + pathId = i.GetWriteMeta().GetPathId().InternalPathId; } else { - AFL_VERIFY(pathId == i.GetWriteMeta().GetTableId()); + AFL_VERIFY(pathId == i.GetWriteMeta().GetPathId().InternalPathId); } } AFL_VERIFY(pathId); diff --git a/ydb/core/tx/columnshard/operations/manager.cpp b/ydb/core/tx/columnshard/operations/manager.cpp index 18adb2263a49..ef7d66aaaa92 100644 --- a/ydb/core/tx/columnshard/operations/manager.cpp +++ b/ydb/core/tx/columnshard/operations/manager.cpp @@ -28,7 +28,7 @@ bool TOperationsManager::Load(NTabletFlatExecutor::TTransactionContext& txc) { NKikimrTxColumnShard::TInternalOperationData metaProto; Y_ABORT_UNLESS(metaProto.ParseFromString(metadata)); - auto operation = std::make_shared(TInternalPathId{}, writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), + auto operation = std::make_shared(TUnifiedPathId{}, writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), granuleShardingVersionId, NEvWrite::EModificationType::Upsert, false); operation->FromProto(metaProto); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_TX)("event", "register_operation_on_load")("operation_id", operation->GetWriteId()); @@ -202,7 +202,7 @@ void TOperationsManager::LinkTransactionOnExecute(const ui64 lockId, const ui64 void TOperationsManager::LinkTransactionOnComplete(const ui64 /*lockId*/, const ui64 /*txId*/) { } -TWriteOperation::TPtr TOperationsManager::RegisterOperation(const TInternalPathId pathId, const ui64 lockId, const ui64 cookie, +TWriteOperation::TPtr TOperationsManager::RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool portionsWriting) { auto writeId = BuildNextOperationWriteId(); auto operation = std::make_shared(pathId, writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), diff --git a/ydb/core/tx/columnshard/operations/manager.h b/ydb/core/tx/columnshard/operations/manager.h index ca74642cffa3..3c381498c7c8 100644 --- a/ydb/core/tx/columnshard/operations/manager.h +++ b/ydb/core/tx/columnshard/operations/manager.h @@ -199,7 +199,7 @@ class TOperationsManager { return *result; } - TWriteOperation::TPtr RegisterOperation(const TInternalPathId pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, + TWriteOperation::TPtr RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool portionsWriting); bool RegisterLock(const ui64 lockId, const ui64 generationId) { if (LockFeatures.contains(lockId)) { diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp index a7acad8b6d91..34429f20b351 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp @@ -103,7 +103,7 @@ class TPortionWriteController: public NColumnShard::IWriteController, void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { const NActors::TLogContextGuard g = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletId)( "parent_id", Context.GetTabletActorId())("write_id", WriteData.GetWriteMeta().GetWriteId())( - "table_id", WriteData.GetWriteMeta().GetTableId()); + "path_id", WriteData.GetWriteMeta().GetPathId()); if (!Context.IsActive()) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "abort_execution"); ReplyError("execution aborted", NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Internal); @@ -133,7 +133,7 @@ void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { continue; } auto portionConclusion = - Context.GetActualSchema()->PrepareForWrite(Context.GetActualSchema(), WriteData.GetWriteMeta().GetTableId(), batch, + Context.GetActualSchema()->PrepareForWrite(Context.GetActualSchema(), WriteData.GetWriteMeta().GetPathId().InternalPathId, batch, WriteData.GetWriteMeta().GetModificationType(), Context.GetStoragesManager(), Context.GetSplitterCounters()); if (portionConclusion.IsFail()) { ReplyError(portionConclusion.GetErrorMessage(), NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Request); diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 68260095e2f4..11e6c9d4ff92 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -14,7 +14,7 @@ namespace NKikimr::NColumnShard { -TWriteOperation::TWriteOperation(const TInternalPathId pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, +TWriteOperation::TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool writePortions) : PathId(pathId) @@ -33,7 +33,8 @@ void TWriteOperation::Start( Y_ABORT_UNLESS(Status == EOperationStatus::Draft); auto writeMeta = std::make_shared( - (ui64)WriteId, GetPathId(), source, GranuleShardingVersionId, GetIdentifier(), context.GetWritingCounters()->GetWriteFlowCounters()); + (ui64)WriteId, PathId, source, GranuleShardingVersionId, GetIdentifier(), + context.GetWritingCounters()->GetWriteFlowCounters()); writeMeta->SetLockId(LockId); writeMeta->SetModificationType(ModificationType); NEvWrite::TWriteData writeData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), @@ -62,7 +63,7 @@ void TWriteOperation::CommitOnExecute( } } else { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).CommitPortionOnExecute(txc, i, snapshot); + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnExecute(txc, i, snapshot); } } } @@ -73,7 +74,7 @@ void TWriteOperation::CommitOnComplete(TColumnShard& owner, const NOlap::TSnapsh owner.UpdateInsertTableCounters(); } else { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).CommitPortionOnComplete( + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnComplete( i, owner.MutableIndexAs()); } } @@ -110,7 +111,7 @@ void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& prot } proto.SetModificationType((ui32)ModificationType); proto.SetWritePortions(WritePortions); - proto.SetPathId(PathId.GetRawValue()); + PathId.InternalPathId.ToProto(proto); } void TWriteOperation::FromProto(const NKikimrTxColumnShard::TInternalOperationData& proto) { @@ -118,8 +119,8 @@ void TWriteOperation::FromProto(const NKikimrTxColumnShard::TInternalOperationDa InsertWriteIds.push_back(TInsertWriteId(writeId)); } WritePortions = proto.GetWritePortions(); - PathId = TInternalPathId::FromRawValue(proto.GetPathId()); - AFL_VERIFY(!WritePortions || PathId); + PathId.InternalPathId = TInternalPathId::FromProto(proto); + AFL_VERIFY(!WritePortions || PathId.InternalPathId); if (proto.HasModificationType()) { ModificationType = (NEvWrite::EModificationType)proto.GetModificationType(); } else { @@ -139,7 +140,7 @@ void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::T owner.InsertTable->Abort(dbTable, writeIds); } else { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnExecute( + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnExecute( txc, i, owner.GetCurrentSnapshotForInternalModification()); } } @@ -149,7 +150,7 @@ void TWriteOperation::AbortOnComplete(TColumnShard& owner) const { Y_ABORT_UNLESS(Status != EOperationStatus::Draft); if (WritePortions) { for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId).AbortPortionOnComplete( + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnComplete( i, owner.MutableIndexAs()); } } diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index 97800d78dc8a..d9276d26785d 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -50,7 +50,7 @@ enum class EOperationBehaviour : ui32 { class TWriteOperation: public TMonitoringObjectsCounter { private: YDB_READONLY(TString, Identifier, TGUID::CreateTimebased().AsGuidString()); - YDB_READONLY_DEF(TInternalPathId, PathId); + YDB_READONLY_DEF(TUnifiedPathId, PathId); YDB_READONLY(EOperationStatus, Status, EOperationStatus::Draft); YDB_READONLY_DEF(TInstant, CreatedAt); YDB_READONLY_DEF(TOperationWriteId, WriteId); @@ -70,7 +70,7 @@ class TWriteOperation: public TMonitoringObjectsCounter { *Activity = 0; } - TWriteOperation(const TInternalPathId pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, + TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool writePortions); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 7e38d70bb99d..08ba00ef3ee4 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -18,6 +18,26 @@ void TSchemaPreset::Deserialize(const NKikimrSchemeOp::TColumnTableSchemaPreset& Name = presetProto.GetName(); } + +std::optional TTablesManager::ResolveSchemeShardLocalPathId(const TInternalPathId internalPathId) const { + if (!HasTable(internalPathId)) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "resolve_ss_path_id")("internal", internalPathId)("result", "not_found"); + return std::nullopt; + } + const auto p = Tables.FindPtr(internalPathId); + AFL_VERIFY(p)("internal", internalPathId); + return p->GetPathId().SchemeShardLocalPathId; +} + +std::optional TTablesManager::ResolveInternalPathId(const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId) const { + if (const auto* internalPathId = SchemeShardLocalToInternal.FindPtr(schemeShardLocalPathId)) { + return {*internalPathId}; + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "resolve_internal_path_id")("ss_local", schemeShardLocalPathId)("result", "not_found"); + return std::nullopt; + } +} + bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json) { NIceDb::TNiceDb db(txc.DB); { @@ -56,10 +76,11 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { while (!rowset.EndOfSet()) { TTableInfo table = table.InitFromDB(rowset); if (table.IsDropped()) { - AFL_VERIFY(PathsToDrop[table.GetDropVersionVerified()].emplace(table.GetPathId()).second); + AFL_VERIFY(PathsToDrop[table.GetDropVersionVerified()].emplace(table.GetPathId().InternalPathId).second); } - - AFL_VERIFY(Tables.emplace(table.GetPathId(), std::move(table)).second); + const auto& pathId = table.GetPathId(); + AFL_VERIFY(Tables.emplace(pathId.InternalPathId, std::move(table)).second); + AFL_VERIFY(SchemeShardLocalToInternal.emplace(pathId.SchemeShardLocalPathId, pathId.InternalPathId).second); if (!rowset.Next()) { timer.AddLoadingFail(); @@ -182,6 +203,17 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { return true; } + +THashMap TTablesManager::ResolveInternalPathIds(const TSchemeShardLocalPathId from, const TSchemeShardLocalPathId to) const { + THashMap result; + for (const auto& [schemeShardLocalPathId, internalPathId]: SchemeShardLocalToInternal) { + if ((from <= schemeShardLocalPathId) && (schemeShardLocalPathId <= to)) { + result.emplace(schemeShardLocalPathId, internalPathId); + } + } + return result; +} + bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDeleted, const std::optional minReadSnapshot) const { auto it = Tables.find(pathId); if (it == Tables.end()) { @@ -193,6 +225,12 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet return true; } +TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) { + const auto result = TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue() + InternalPathIdOffset); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "CreateInternalPathId")("schemeShardLocalPathId", schemeShardLocalPathId)("result", result); + return result; +} + bool TTablesManager::IsReadyForStartWrite(const TInternalPathId pathId, const bool withDeleted) const { return HasPrimaryIndex() && HasTable(pathId, withDeleted); } @@ -233,13 +271,16 @@ void TTablesManager::DropPreset(const ui32 presetId, const NOlap::TSnapshot& ver } void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { - Y_ABORT_UNLESS(!HasTable(table.GetPathId())); + Y_ABORT_UNLESS(!HasTable(table.GetPathId().InternalPathId)); Y_ABORT_UNLESS(table.IsEmpty()); + NYDBTest::TControllers::GetColumnShardController()->OnAddPathId(TabletId, table.GetPathId()); - Schema::SaveTableInfo(db, table.GetPathId()); - const auto pathId = table.GetPathId(); + Schema::SaveTableInfo(db, table.GetPathId().InternalPathId); + const auto pathId = table.GetPathId().InternalPathId; AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("method", "RegisterTable")("path_id", pathId); AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size()); + AFL_VERIFY(SchemeShardLocalToInternal.emplace(table.GetPathId().SchemeShardLocalPathId, table.GetPathId().InternalPathId).second); + Schema::SaveTableSchemeShardLocalPathId(db, table.GetPathId().InternalPathId, table.GetPathId().SchemeShardLocalPathId); if (PrimaryIndex) { PrimaryIndex->RegisterTable(pathId); } @@ -287,8 +328,11 @@ void TTablesManager::AddSchemaVersion( } std::unique_ptr TTablesManager::CreateAddShardingInfoTx( - TColumnShard& owner, const TInternalPathId pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const { - return std::make_unique(owner, tabletShardingLogic, pathId, versionId); + TColumnShard& owner, const TSchemeShardLocalPathId schemeShardLocalPathId, const ui64 versionId, + const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const { + const auto* internalPathId = SchemeShardLocalToInternal.FindPtr(schemeShardLocalPathId); + AFL_VERIFY(internalPathId)("scheme_shard_local_path_id", schemeShardLocalPathId); + return std::make_unique(owner, tabletShardingLogic, *internalPathId, versionId); } void TTablesManager::AddTableVersion(const TInternalPathId pathId, const NOlap::TSnapshot& version, @@ -325,6 +369,14 @@ void TTablesManager::AddTableVersion(const TInternalPathId pathId, const NOlap:: table.AddVersion(version); } +namespace { + +ui64 GetInternalPathIdOffset(ui64 tabletId) { + return NYDBTest::TControllers::GetColumnShardController()->GetInternalPathIdOffset(tabletId); +} + +} //namespace + TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const std::shared_ptr& dataAccessorsManager, const std::shared_ptr& schemaCache, const std::shared_ptr& portionsStats, @@ -334,7 +386,8 @@ TTablesManager::TTablesManager(const std::shared_ptr& s , LoadTimeCounters(std::make_unique()) , SchemaObjectsCache(schemaCache) , PortionsStats(portionsStats) - , TabletId(tabletId) { + , TabletId(tabletId) + , InternalPathIdOffset(GetInternalPathIdOffset(tabletId)) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const TInternalPathId pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 209ae1e747aa..0d2ef20f8659 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -93,7 +93,8 @@ class TSchemaPreset: public TVersionedSchema DropVersion; YDB_READONLY_DEF(TSet, Versions); @@ -102,8 +103,8 @@ class TTableInfo { return Versions.empty(); } - TInternalPathId GetPathId() const { - return PathId; + TUnifiedPathId GetPathId() const { + return TUnifiedPathId{InternalPathId, SchemeShardLocalPathId}; } const NOlap::TSnapshot& GetDropVersionVerified() const { @@ -130,14 +131,22 @@ class TTableInfo { return *DropVersion < *minReadSnapshot; } - TTableInfo(const TInternalPathId pathId) - : PathId(pathId) { + TTableInfo(const TUnifiedPathId& pathId) + : InternalPathId(pathId.InternalPathId) + , SchemeShardLocalPathId(pathId.SchemeShardLocalPathId) { } template static TTableInfo InitFromDB(const TRow& rowset) { - const auto pathId = TInternalPathId::FromRawValue(rowset.template GetValue()); - TTableInfo result(pathId); + const auto internalPathId = TInternalPathId::FromRawValue(rowset.template GetValue()); + AFL_VERIFY(internalPathId); + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromRawValue( + rowset.template HaveValue() + ? rowset.template GetValue() + : internalPathId.GetRawValue() + ); + AFL_VERIFY(schemeShardLocalPathId); + TTableInfo result({internalPathId, schemeShardLocalPathId}); if (rowset.template HaveValue() && rowset.template HaveValue()) { result.DropVersion.emplace( rowset.template GetValue(), rowset.template GetValue()); @@ -186,9 +195,10 @@ class TTtlVersions { } }; -class TTablesManager { +class TTablesManager: public NOlap::IPathIdTranslator { private: THashMap Tables; + THashMap SchemeShardLocalToInternal; THashSet SchemaPresetsIds; THashMap ActualSchemaForPreset; std::map> PathsToDrop; @@ -200,10 +210,14 @@ class TTablesManager { NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; ui64 TabletId = 0; + ui64 InternalPathIdOffset; -public: friend class TTxInit; +public: //IPathIdTranslator + virtual std::optional ResolveSchemeShardLocalPathId(const TInternalPathId internalPathId) const override; + virtual std::optional ResolveInternalPathId(const NColumnShard::TSchemeShardLocalPathId schemeShardLocalPathId) const override; +public: TTablesManager(const std::shared_ptr& storagesManager, const std::shared_ptr& dataAccessorsManager, const std::shared_ptr& schemaCache, const std::shared_ptr& portionsStats, @@ -309,7 +323,8 @@ class TTablesManager { const TTableInfo& GetTable(const TInternalPathId pathId) const; ui64 GetMemoryUsage() const; - + TInternalPathId CreateInternalPathId(const TSchemeShardLocalPathId schemShardLocalPathId); + THashMap ResolveInternalPathIds(const TSchemeShardLocalPathId from, const TSchemeShardLocalPathId to) const; bool HasTable(const TInternalPathId pathId, const bool withDeleted = false, const std::optional minReadSnapshot = std::nullopt) const; bool IsReadyForStartWrite(const TInternalPathId pathId, const bool withDeleted) const; bool IsReadyForFinishWrite(const TInternalPathId pathId, const NOlap::TSnapshot& minReadSnapshot) const; @@ -327,7 +342,7 @@ class TTablesManager { const std::optional& schema, NIceDb::TNiceDb& db); bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); - [[nodiscard]] std::unique_ptr CreateAddShardingInfoTx(TColumnShard& owner, const TInternalPathId pathId, + [[nodiscard]] std::unique_ptr CreateAddShardingInfoTx(TColumnShard& owner, const NColumnShard::TSchemeShardLocalPathId pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const; void SetSchemaObjectsCache(const std::shared_ptr& cache) { diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 93d33460609b..845949cb62b8 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -9,8 +9,8 @@ namespace NKikimr::NColumnShard { bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { owner->Counters.GetCSCounters().WritingCounters->OnWritingTaskDequeue(TMonotonic::Now() - Created); owner->OperationsManager->RegisterLock(LockId, owner->Generation()); - auto writeOperation = owner->OperationsManager->RegisterOperation( - PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType, AppDataVerified().FeatureFlags.GetEnableWritePortionsOnInsert()); + auto writeOperation = owner->OperationsManager->RegisterOperation(PathId, + LockId, Cookie, GranuleShardingVersionId, ModificationType, AppDataVerified().FeatureFlags.GetEnableWritePortionsOnInsert()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", ArrowData->GetSize())("operation_id", writeOperation->GetIdentifier())( "in_flight", owner->Counters.GetWritesMonitor()->GetWritesInFlight())( @@ -22,7 +22,7 @@ bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { owner->Counters.GetIndexationCounters().SplitterCounters, owner->Counters.GetCSCounters().WritingCounters, NOlap::TSnapshot::Max(), writeOperation->GetActivityChecker(), Behaviour == EOperationBehaviour::NoTxWrite, owner->BufferizationInsertionWriteActorId, owner->BufferizationPortionsWriteActorId); - ArrowData->SetSeparationPoints(owner->GetIndexAs().GetGranulePtrVerified(PathId)->GetBucketPositions()); + ArrowData->SetSeparationPoints(owner->GetIndexAs().GetGranulePtrVerified(PathId.InternalPathId)->GetBucketPositions()); writeOperation->Start(*owner, ArrowData, SourceId, wContext); return true; } @@ -63,7 +63,7 @@ bool TWriteTasksQueue::Drain(const bool onWakeup, const TActorContext& ctx) { } void TWriteTasksQueue::Enqueue(TWriteTask&& task) { - const TInternalPathId pathId = task.GetPathId(); + const auto& pathId = task.GetInternalPathId(); WriteTasks[pathId].emplace_back(std::move(task)); } diff --git a/ydb/core/tx/columnshard/tablet/write_queue.h b/ydb/core/tx/columnshard/tablet/write_queue.h index aba35e5a6a20..e86aacf33f88 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.h +++ b/ydb/core/tx/columnshard/tablet/write_queue.h @@ -13,7 +13,7 @@ class TWriteTask: TMoveOnly { NOlap::ISnapshotSchema::TPtr Schema; const NActors::TActorId SourceId; const std::optional GranuleShardingVersionId; - const TInternalPathId PathId; + const TUnifiedPathId PathId; const ui64 Cookie; const ui64 LockId; const NEvWrite::EModificationType ModificationType; @@ -22,7 +22,7 @@ class TWriteTask: TMoveOnly { public: TWriteTask(const std::shared_ptr& arrowData, const NOlap::ISnapshotSchema::TPtr& schema, const NActors::TActorId sourceId, - const std::optional& granuleShardingVersionId, const TInternalPathId pathId, const ui64 cookie, const ui64 lockId, + const std::optional& granuleShardingVersionId, const TUnifiedPathId pathId, const ui64 cookie, const ui64 lockId, const NEvWrite::EModificationType modificationType, const EOperationBehaviour behaviour) : ArrowData(arrowData) , Schema(schema) @@ -35,8 +35,8 @@ class TWriteTask: TMoveOnly { , Behaviour(behaviour) { } - const TInternalPathId& GetPathId() const { - return PathId; + const TInternalPathId& GetInternalPathId() const { + return PathId.InternalPathId; } const TMonotonic& GetCreatedMonotonic() const { diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index a47cfeb985dd..a6cd6f43f272 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -549,7 +549,7 @@ namespace NKikimr::NColumnShard { NTxUT::TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, snapshot); reader.SetReplyColumnIds(fields); auto rb = reader.ReadAll(); - UNIT_ASSERT(reader.IsCorrectlyFinished()); + //UNIT_ASSERT(reader.IsCorrectlyFinished()); return rb ? rb : NArrow::MakeEmptyBatch(NArrow::MakeArrowSchema(schema)); } } diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 58216b25067c..57462809f48f 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -283,7 +284,7 @@ struct TTestSchema { NKikimrTxColumnShard::TSchemaTxBody tx; tx.MutableSeqNo()->SetGeneration(generation); auto* table = tx.MutableEnsureTables()->AddTables(); - table->SetPathId(pathId); + NColumnShard::TSchemeShardLocalPathId::FromRawValue(pathId).ToProto(*table); { // preset auto* preset = table->MutableSchemaPreset(); @@ -309,7 +310,7 @@ struct TTestSchema { auto* table = tx.MutableInitShard()->AddTables(); tx.MutableInitShard()->SetOwnerPath(ownerPath); tx.MutableInitShard()->SetOwnerPathId(pathId); - table->SetPathId(pathId); + NColumnShard::TSchemeShardLocalPathId::FromRawValue(pathId).ToProto(*table); { // preset auto* preset = table->MutableSchemaPreset(); @@ -335,7 +336,7 @@ struct TTestSchema { auto* table = tx.MutableInitShard()->AddTables(); tx.MutableInitShard()->SetOwnerPath(path); tx.MutableInitShard()->SetOwnerPathId(pathId); - table->SetPathId(pathId); + NColumnShard::TSchemeShardLocalPathId::FromRawValue(pathId).ToProto(*table); InitSchema(columns, pk, specials, table->MutableSchema()); InitTiersAndTtl(specials, table->MutableTtlSettings()); @@ -351,7 +352,7 @@ struct TTestSchema { const std::vector& pk, const TTableSpecials& specials) { NKikimrTxColumnShard::TSchemaTxBody tx; auto* table = tx.MutableAlterTable(); - table->SetPathId(pathId); + NColumnShard::TSchemeShardLocalPathId::FromRawValue(pathId).ToProto(*table); tx.MutableSeqNo()->SetRound(version); auto* preset = table->MutableSchemaPreset(); @@ -373,7 +374,7 @@ struct TTestSchema { static TString DropTableTxBody(ui64 pathId, ui32 version) { NKikimrTxColumnShard::TSchemaTxBody tx; - tx.MutableDropTable()->SetPathId(pathId); + NColumnShard::TSchemeShardLocalPathId::FromRawValue(pathId).ToProto(*tx.MutableDropTable()); tx.MutableSeqNo()->SetRound(version); TString out; diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.cpp b/ydb/core/tx/columnshard/transactions/operators/schema.cpp index 0f79afc374d6..38726fb0e29f 100644 --- a/ydb/core/tx/columnshard/transactions/operators/schema.cpp +++ b/ydb/core/tx/columnshard/transactions/operators/schema.cpp @@ -171,9 +171,11 @@ void TSchemaTransactionOperator::DoOnTabletInit(TColumnShard& owner) { case NKikimrTxColumnShard::TSchemaTxBody::kEnsureTables: { for (auto&& i : SchemaTxBody.GetEnsureTables().GetTables()) { - const auto pathId = TInternalPathId::FromRawValue(i.GetPathId()); - if (owner.TablesManager.HasTable(pathId, true) && !owner.TablesManager.HasTable(pathId)) { - WaitPathIdsToErase.emplace(pathId); + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(i); + if (const auto internalPathId = owner.TablesManager.ResolveInternalPathId(schemeShardLocalPathId)) { + if (owner.TablesManager.HasTable(*internalPathId, true)) { + WaitPathIdsToErase.emplace(*internalPathId); + } } } } diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.h b/ydb/core/tx/columnshard/transactions/operators/schema.h index 2a7409479f40..07d1f6e2f917 100644 --- a/ydb/core/tx/columnshard/transactions/operators/schema.h +++ b/ydb/core/tx/columnshard/transactions/operators/schema.h @@ -25,9 +25,11 @@ class TSchemaTransactionOperator: public IProposeTxOperator, public TMonitoringO THashSet GetNotErasedTableIds(const TColumnShard& owner, const TInfoProto& tables) const { THashSet result; for (auto&& i : tables) { - const auto& pathId = TInternalPathId::FromRawValue(i.GetPathId()); - if (owner.TablesManager.HasTable(pathId, true)) { - result.emplace(pathId); + const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(i); + if (const auto internalPathId = owner.TablesManager.ResolveInternalPathId(schemeShardLocalPathId)) { + if (owner.TablesManager.HasTable(*internalPathId, true)) { + result.emplace(*internalPathId); + } } } if (result.size()) { @@ -74,7 +76,7 @@ class TSchemaTransactionOperator: public IProposeTxOperator, public TMonitoringO return false; } TxAddSharding = owner.TablesManager.CreateAddShardingInfoTx( - owner, TInternalPathId::FromRawValue(SchemaTxBody.GetGranuleShardingInfo().GetPathId()), SchemaTxBody.GetGranuleShardingInfo().GetVersionId(), infoContainer); + owner, TSchemeShardLocalPathId::FromProto(SchemaTxBody.GetGranuleShardingInfo()), SchemaTxBody.GetGranuleShardingInfo().GetVersionId(), infoContainer); } return true; } diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index 09a74a986b2a..e1bfabe4514c 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -137,20 +137,20 @@ class TTablesCleaner: public NYDBTest::ILocalDBModifier { using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); - std::vector tables; + std::vector tables; { auto rowset = db.Table().Select(); UNIT_ASSERT(rowset.IsReady()); while (!rowset.EndOfSet()) { - const auto pathId = rowset.GetValue(); + const auto pathId = TInternalPathId::FromRawValue(rowset.GetValue()); tables.emplace_back(pathId); UNIT_ASSERT(rowset.Next()); } } for (auto&& key : tables) { - db.Table().Key(key).Delete(); + db.Table().Key(key.GetRawValue()).Delete(); } struct TKey { diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index 33af6e744eb1..0f005acb826d 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -37,7 +37,7 @@ class IDataContainer { class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TNonCopyable { private: YDB_ACCESSOR(ui64, WriteId, 0); - YDB_READONLY_DEF(NColumnShard::TInternalPathId, TableId); + YDB_READONLY_DEF(NColumnShard::TUnifiedPathId, PathId); YDB_ACCESSOR_DEF(NActors::TActorId, Source); YDB_ACCESSOR_DEF(std::optional, GranuleShardingVersion); YDB_READONLY(TString, Id, TGUID::CreateTimebased().AsUuidString()); @@ -88,10 +88,10 @@ class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TN } } - TWriteMeta(const ui64 writeId, const NColumnShard::TInternalPathId tableId, const NActors::TActorId& source, const std::optional granuleShardingVersion, + TWriteMeta(const ui64 writeId, const NColumnShard::TUnifiedPathId& pathId, const NActors::TActorId& source, const std::optional granuleShardingVersion, const TString& writingIdentifier, const std::shared_ptr& counters) : WriteId(writeId) - , TableId(tableId) + , PathId(pathId) , Source(source) , GranuleShardingVersion(granuleShardingVersion) , Id(writingIdentifier) From 86ece5a8f5aed9a51f5d04d97bb9eb6025a86b06 Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Fri, 13 Jun 2025 14:00:05 +0300 Subject: [PATCH 17/62] Add InternalPathId column to primary_index_granule_stats (#19644) --- ydb/core/kqp/ut/olap/helpers/local.h | 2 +- ydb/core/kqp/ut/olap/indexes_ut.cpp | 2 +- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 2 +- ydb/core/kqp/ut/olap/sys_view_ut.cpp | 109 ++++++++++++++++++ ydb/core/kqp/ut/olap/tiering_ut.cpp | 2 +- ydb/core/sys_view/common/schema.h | 4 +- .../reader/sys_view/granules/granules.cpp | 1 + 7 files changed, 117 insertions(+), 5 deletions(-) diff --git a/ydb/core/kqp/ut/olap/helpers/local.h b/ydb/core/kqp/ut/olap/helpers/local.h index 0ac389f99441..1e04b4ca4c7c 100644 --- a/ydb/core/kqp/ut/olap/helpers/local.h +++ b/ydb/core/kqp/ut/olap/helpers/local.h @@ -36,7 +36,7 @@ class TLocalHelper: public Tests::NCS::THelper { CreateOlapTablesWithStore(tableNames, storeName, storeShardsCount, tableShardsCount); } - void CreateTestOlapTableWithoutStore(TString tableName = "olapTable", ui32 tableShardsCount = 3) { + void CreateTestOlapStandaloneTable(TString tableName = "olapTable", ui32 tableShardsCount = 3) { CreateOlapTables({tableName}, tableShardsCount); } diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp index 8152ccba8c83..5c9227d5adf0 100644 --- a/ydb/core/kqp/ut/olap/indexes_ut.cpp +++ b/ydb/core/kqp/ut/olap/indexes_ut.cpp @@ -156,7 +156,7 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) { csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); csController->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings()); - TLocalHelper(kikimr).CreateTestOlapTableWithoutStore(); + TLocalHelper(kikimr).CreateTestOlapStandaloneTable(); auto tableClient = kikimr.GetTableClient(); auto& client = kikimr.GetTestClient(); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index e3bbcf181daa..203f1a900314 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -220,7 +220,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { auto settings = TKikimrSettings() .SetWithSampleTables(false); TKikimrRunner kikimr(settings); - TLocalHelper(kikimr).CreateTestOlapTableWithoutStore(); + TLocalHelper(kikimr).CreateTestOlapStandaloneTable(); { //1. QueryService diff --git a/ydb/core/kqp/ut/olap/sys_view_ut.cpp b/ydb/core/kqp/ut/olap/sys_view_ut.cpp index d74c64b227c4..46e9d7f78e3b 100644 --- a/ydb/core/kqp/ut/olap/sys_view_ut.cpp +++ b/ydb/core/kqp/ut/olap/sys_view_ut.cpp @@ -12,6 +12,115 @@ namespace NKikimr::NKqp { Y_UNIT_TEST_SUITE(KqpOlapSysView) { + + + Y_UNIT_TEST(GranulePathId_Store) { + auto settings = TKikimrSettings().SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + auto helper = TLocalHelper(kikimr); + const ui32 storeShardsCount = 101; + const ui32 tableShardsCount = 17; + const ui32 tablesCount = 1013; + TVector tableNames; + for (ui32 i = 0; i < tablesCount; ++i) { + tableNames.push_back(TStringBuilder() << "table" << i); + } + helper.CreateTestOlapTables(tableNames, "columnStore", storeShardsCount, tableShardsCount); + const auto tablets = csController->GetActiveTablets(); + UNIT_ASSERT_VALUES_EQUAL(tablets.size(), storeShardsCount); + + auto tableClient = kikimr.GetTableClient(); + + { + //check the store + auto selectQuery = TString(R"( + SELECT PathId, TabletId, InternalPathId, + FROM `/Root/columnStore/.sys/store_primary_index_granule_stats` + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery, true); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount * tablesCount); + THashMap> result; + for (const auto& row : rows) { + result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] = + NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId"))); + } + UNIT_ASSERT_VALUES_EQUAL(result.size(), tablesCount); + + for (const auto& [tabletId, pathIdTranslator] : tablets) { + const auto& pathIds = pathIdTranslator->GetSchemeShardLocalPathIds(); + for (const auto& pathId : pathIds) { + const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId); + UNIT_ASSERT(internalPathId.has_value()); + UNIT_ASSERT(result.contains(pathId) && result[pathId].contains(tabletId)); + UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId); + } + } + } + + { + //check a table in the store + auto selectQuery = TString(R"( + SELECT PathId, TabletId, InternalPathId, + FROM `/Root/columnStore/table2/.sys/primary_index_granule_stats` + )"); + + auto rows = ExecuteScanQuery(tableClient, selectQuery, true); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount); + THashMap> result; + for (const auto& row : rows) { + result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] = + NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId"))); + } + UNIT_ASSERT_VALUES_EQUAL(result.size(), 1); + const auto& pathId = result.begin()->first; + UNIT_ASSERT_VALUES_EQUAL(result[pathId].size(), tableShardsCount); + + for (const auto& [tabletId, pathIdTranslator] : tablets) { + if (const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId)) { + UNIT_ASSERT(result[pathId].contains(tabletId)); + UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId); + } + } + } + } + + Y_UNIT_TEST(GranulePathId_Standalone) { + auto settings = TKikimrSettings().SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + auto helper = TLocalHelper(kikimr); + const ui32 tableShardsCount = 1201; + helper.CreateTestOlapStandaloneTable("table", tableShardsCount); + const auto tablets = csController->GetActiveTablets(); + UNIT_ASSERT_VALUES_EQUAL(tablets.size(), tableShardsCount); + auto tableClient = kikimr.GetTableClient(); + auto selectQuery = TString(R"( + SELECT PathId, TabletId, InternalPathId, + FROM `/Root/table/.sys/primary_index_granule_stats` + )"); + auto rows = ExecuteScanQuery(tableClient, selectQuery, true); + UNIT_ASSERT_VALUES_EQUAL(rows.size(), tableShardsCount); + THashMap> result; + for (const auto& row : rows) { + result[NColumnShard::TSchemeShardLocalPathId::FromRawValue(GetUint64(row.at("PathId")))][GetUint64(row.at("TabletId"))] = + NColumnShard::TInternalPathId::FromRawValue(GetUint64(row.at("InternalPathId"))); + } + UNIT_ASSERT_VALUES_EQUAL(result.size(), 1); + const auto& pathId = result.begin()->first; + UNIT_ASSERT_VALUES_EQUAL(result[pathId].size(), tableShardsCount); + + for (const auto& [tabletId, pathIdTranslator] : tablets) { + const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId); + UNIT_ASSERT(internalPathId.has_value()); + UNIT_ASSERT(result[pathId].contains(tabletId)); + UNIT_ASSERT_VALUES_EQUAL(result[pathId][tabletId], *internalPathId); + } + } + Y_UNIT_TEST(StatsSysView) { auto settings = TKikimrSettings() .SetWithSampleTables(false).SetColumnShardAlterObjectEnabled(true); diff --git a/ydb/core/kqp/ut/olap/tiering_ut.cpp b/ydb/core/kqp/ut/olap/tiering_ut.cpp index 401ee5874353..44536d2e83a4 100644 --- a/ydb/core/kqp/ut/olap/tiering_ut.cpp +++ b/ydb/core/kqp/ut/olap/tiering_ut.cpp @@ -164,7 +164,7 @@ Y_UNIT_TEST_SUITE(KqpOlapTiering) { auto& testHelper = tieringHelper.GetTestHelper(); tieringHelper.SetTablePath("/Root/olapTable"); - olapHelper.CreateTestOlapTableWithoutStore(); + olapHelper.CreateTestOlapStandaloneTable(); testHelper.CreateTier("tier1"); testHelper.SetTiering("/Root/olapTable", DEFAULT_TIER_NAME, DEFAULT_COLUMN_NAME); { diff --git a/ydb/core/sys_view/common/schema.h b/ydb/core/sys_view/common/schema.h index 4bd52bccae60..2e7120140507 100644 --- a/ydb/core/sys_view/common/schema.h +++ b/ydb/core/sys_view/common/schema.h @@ -596,6 +596,7 @@ struct Schema : NIceDb::Schema { struct PortionsCount: Column<3, NScheme::NTypeIds::Uint64> {}; struct HostName: Column<4, NScheme::NTypeIds::Utf8> {}; struct NodeId: Column<5, NScheme::NTypeIds::Uint64> {}; + struct InternalPathId: Column<6, NScheme::NTypeIds::Uint64> {}; using TKey = TableKey; using TColumns = TableColumns< @@ -603,7 +604,8 @@ struct Schema : NIceDb::Schema { TabletId, PortionsCount, HostName, - NodeId + NodeId, + InternalPathId >; }; diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp index 0a58782671bf..3cbbf9c7e591 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/granules/granules.cpp @@ -14,6 +14,7 @@ bool TStatsIterator::AppendStats(const std::vector(*builders[2], granule.GetPortions().size()); NArrow::Append(*builders[3], HostNameField); NArrow::Append(*builders[4], NActors::TActivationContext::AsActorContext().SelfID.NodeId()); + NArrow::Append(*builders[5], granule.GetPathId().InternalPathId.GetRawValue()); return false; } From f912d95db22b8d806c497de546fd432798bff9d4 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 13 Jun 2025 18:14:15 +0300 Subject: [PATCH 18/62] remove insert table v1 (#19638) Co-authored-by: ivanmorozov333 --- ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp | 5 - ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 8 - ydb/core/kqp/ut/olap/sparsed_ut.cpp | 2 - ydb/core/kqp/ut/olap/write_ut.cpp | 11 +- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 60 +--- ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp | 3 - ydb/core/kqp/ut/tx/kqp_sink_common.h | 3 - .../transaction/tx_blobs_written.cpp | 1 - .../transaction/tx_gc_insert_table.cpp | 37 --- .../transaction/tx_gc_insert_table.h | 25 -- .../blobs_action/transaction/tx_write.cpp | 141 -------- .../blobs_action/transaction/tx_write.h | 52 --- .../transaction/tx_write_index.cpp | 1 - .../blobs_action/transaction/ya.make | 2 - ydb/core/tx/columnshard/columnshard.cpp | 19 +- ydb/core/tx/columnshard/columnshard.h | 51 --- ydb/core/tx/columnshard/columnshard__init.cpp | 5 - .../columnshard/columnshard__progress_tx.cpp | 1 - .../tx/columnshard/columnshard__write.cpp | 38 +-- ydb/core/tx/columnshard/columnshard_impl.cpp | 211 +----------- ydb/core/tx/columnshard/columnshard_impl.h | 41 --- .../tx/columnshard/columnshard_schema.cpp | 28 -- ydb/core/tx/columnshard/columnshard_schema.h | 185 ----------- .../tx/columnshard/counters/columnshard.cpp | 2 - .../tx/columnshard/counters/columnshard.h | 8 - .../columnshard/counters/counters_manager.h | 5 - .../tx/columnshard/counters/insert_table.cpp | 16 - .../tx/columnshard/counters/insert_table.h | 72 ----- .../tx/columnshard/counters/tablet_counters.h | 13 - ydb/core/tx/columnshard/counters/ya.make | 1 - .../engines/changes/cleanup_tables.cpp | 2 - .../engines/changes/indexation.cpp | 300 ------------------ .../columnshard/engines/changes/indexation.h | 63 ---- .../tx/columnshard/engines/changes/ya.make | 2 - .../tx/columnshard/engines/column_engine.h | 1 - .../engines/column_engine_logs.cpp | 22 -- .../columnshard/engines/column_engine_logs.h | 1 - .../tx/columnshard/engines/db_wrapper.cpp | 35 -- ydb/core/tx/columnshard/engines/db_wrapper.h | 20 -- .../engines/insert_table/committed.cpp | 5 - .../engines/insert_table/committed.h | 163 ---------- .../engines/insert_table/insert_table.cpp | 218 ------------- .../engines/insert_table/insert_table.h | 129 -------- .../engines/insert_table/inserted.cpp | 12 - .../engines/insert_table/inserted.h | 35 -- .../columnshard/engines/insert_table/meta.cpp | 28 -- .../columnshard/engines/insert_table/meta.h | 59 ---- .../engines/insert_table/path_info.cpp | 75 ----- .../engines/insert_table/path_info.h | 86 ----- .../engines/insert_table/rt_insertion.cpp | 164 ---------- .../engines/insert_table/rt_insertion.h | 228 ------------- .../engines/insert_table/user_data.cpp | 45 --- .../engines/insert_table/user_data.h | 101 ------ .../columnshard/engines/insert_table/ya.make | 21 -- .../engines/reader/abstract/read_metadata.cpp | 12 +- .../engines/reader/abstract/read_metadata.h | 6 +- .../engines/reader/abstract/ya.make | 1 - .../engines/reader/common/stats.cpp | 1 - .../columnshard/engines/reader/common/stats.h | 1 - .../constructor/read_metadata.cpp | 7 +- .../plain_reader/constructor/constructor.cpp | 5 +- .../constructor/read_metadata.cpp | 16 +- .../plain_reader/constructor/read_metadata.h | 5 +- .../plain_reader/iterator/plain_read_data.cpp | 32 +- .../reader/plain_reader/iterator/source.cpp | 56 ---- .../reader/plain_reader/iterator/source.h | 103 ------ .../simple_reader/constructor/constructor.cpp | 5 +- .../iterator/plain_read_data.cpp | 6 +- ydb/core/tx/columnshard/engines/ut/helper.cpp | 8 +- ydb/core/tx/columnshard/engines/ut/helper.h | 6 +- .../engines/ut/ut_insert_table.cpp | 131 -------- .../columnshard/engines/ut/ut_logs_engine.cpp | 45 --- ydb/core/tx/columnshard/engines/ut/ya.make | 2 - .../writer/indexed_blob_constructor.cpp | 20 -- .../engines/writer/indexed_blob_constructor.h | 2 - ydb/core/tx/columnshard/engines/ya.make | 1 - .../tx/columnshard/hooks/abstract/abstract.h | 1 - .../columnshard/hooks/testing/controller.cpp | 1 - .../hooks/testing/ro_controller.cpp | 7 - .../columnshard/hooks/testing/ro_controller.h | 15 - .../columnshard/inflight_request_tracker.cpp | 29 +- ydb/core/tx/columnshard/loading/stages.cpp | 55 ---- ydb/core/tx/columnshard/loading/stages.h | 19 -- .../normalizer/abstract/abstract.h | 2 +- ydb/core/tx/columnshard/normalizer/ya.make | 1 - .../operations/batch_builder/builder.cpp | 4 +- .../operations/batch_builder/restore.cpp | 2 +- .../tx/columnshard/operations/manager.cpp | 6 +- ydb/core/tx/columnshard/operations/manager.h | 2 +- .../operations/slice_builder/builder.cpp | 117 +++---- .../operations/slice_builder/pack_builder.h | 1 - ydb/core/tx/columnshard/operations/write.cpp | 57 +--- ydb/core/tx/columnshard/operations/write.h | 4 +- .../tx/columnshard/tablet/write_queue.cpp | 3 +- .../columnshard/test_helper/controllers.cpp | 1 - .../transactions/operators/long_tx_write.cpp | 56 ---- .../transactions/operators/long_tx_write.h | 104 ------ .../transactions/operators/ya.make | 1 - .../ut_rw/ut_columnshard_read_write.cpp | 58 ++-- .../ut_schema/ut_columnshard_schema.cpp | 9 +- ydb/core/tx/data_events/write_data.cpp | 5 +- ydb/core/tx/data_events/write_data.h | 3 +- ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 1 - 103 files changed, 152 insertions(+), 3745 deletions(-) delete mode 100644 ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp delete mode 100644 ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.h delete mode 100644 ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp delete mode 100644 ydb/core/tx/columnshard/blobs_action/transaction/tx_write.h delete mode 100644 ydb/core/tx/columnshard/counters/insert_table.cpp delete mode 100644 ydb/core/tx/columnshard/counters/insert_table.h delete mode 100644 ydb/core/tx/columnshard/engines/changes/indexation.cpp delete mode 100644 ydb/core/tx/columnshard/engines/changes/indexation.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/committed.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/committed.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/insert_table.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/inserted.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/inserted.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/meta.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/meta.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/path_info.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/path_info.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/rt_insertion.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/rt_insertion.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/user_data.cpp delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/user_data.h delete mode 100644 ydb/core/tx/columnshard/engines/insert_table/ya.make delete mode 100644 ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp delete mode 100644 ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp delete mode 100644 ydb/core/tx/columnshard/transactions/operators/long_tx_write.h diff --git a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp index 498412b9370d..2e0606d02206 100644 --- a/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp +++ b/ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp @@ -354,15 +354,10 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) { CheckCount(230000); i64 count = CSController->GetShardingFiltersCount().Val(); AFL_VERIFY(count >= 16)("count", count); - CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - CSController->WaitIndexation(TDuration::Seconds(3)); CSController->WaitCompactions(TDuration::Seconds(3)); WriteTestData(Kikimr, "/Root/olapStore/olapTable", 1000000, 300000000, 10000); CheckCount(230000); - CSController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - CSController->WaitIndexation(TDuration::Seconds(5)); - CheckCount(230000); CSController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); CSController->WaitCompactions(TDuration::Seconds(5)); count = CSController->GetShardingFiltersCount().Val(); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 203f1a900314..9d026a8910c7 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -2897,10 +2897,6 @@ Y_UNIT_TEST_SUITE(KqpOlap) { tableInserter.AddRow().Add(2).Add("test_res_2").Add("val2").AddNull(); testHelper.BulkUpsert(testTable, tableInserter); } -// while (csController->GetCompactionFinishedCounter().Val() == 0) { -// Cout << "Wait indexation..." << Endl; -// Sleep(TDuration::Seconds(2)); -// } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=2", "[[2;\"test_res_2\";#;[\"val2\"]]]"); } @@ -3364,7 +3360,6 @@ Y_UNIT_TEST_SUITE(KqpOlap) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); testHelper.CreateTestOlapTable(); auto tableClient = kikimr.GetTableClient(); @@ -3387,9 +3382,6 @@ Y_UNIT_TEST_SUITE(KqpOlap) { UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); } - csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); - } Y_UNIT_TEST(MultiInsertWithSinks) { diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index c18120917501..8da029edd9fa 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -214,7 +214,6 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { } void Execute() { - CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); @@ -350,7 +349,6 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { Tests::NCommon::TLoggerInit(kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD }, "CS").SetPriority(NActors::NLog::PRI_DEBUG).Initialize(); WriteTestData(kikimr, "/Root/olapStore/olapTable", 1000000, 300000000, 10000); - csController->WaitIndexation(TDuration::Seconds(3)); { auto result = session.ExecuteSchemeQuery("ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=ALTER_COLUMN, NAME=uid, `DATA_ACCESSOR_CONSTRUCTOR.CLASS_NAME`=`SPARSED`)").GetValueSync(); diff --git a/ydb/core/kqp/ut/olap/write_ut.cpp b/ydb/core/kqp/ut/olap/write_ut.cpp index 5ce1df3993f8..024d34fec27a 100644 --- a/ydb/core/kqp/ut/olap/write_ut.cpp +++ b/ydb/core/kqp/ut/olap/write_ut.cpp @@ -58,7 +58,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { WriteTestData(kikimr, "/Root/olapStore/olapTable", 30000, 1000000, 11000); WriteTestData(kikimr, "/Root/olapStore/olapTable", 30000, 1000000, 11000); while (csController->GetCompactionStartedCounter().Val() == 0) { - Cout << "Wait indexation..." << Endl; + Cout << "Wait compaction..." << Endl; Sleep(TDuration::Seconds(2)); } while (!Singleton()->GetWritesCount() || @@ -67,7 +67,6 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { << csController->GetIndexWriteControllerBrokeCount().Val() << Endl; Sleep(TDuration::Seconds(2)); } - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); const auto startInstant = TMonotonic::Now(); while (Singleton()->GetSize() && @@ -83,7 +82,6 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetIndexWriteControllerEnabled(false); csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); auto settings = TKikimrSettings().SetWithSampleTables(false); @@ -97,9 +95,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { WriteTestData(kikimr, "/Root/olapStore/olapTable", 30000, 1000000, 11000); TTypedLocalHelper("Utf8", kikimr).ExecuteSchemeQuery("DROP TABLE `/Root/olapStore/olapTable`;"); - csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); - csController->WaitIndexation(TDuration::Seconds(5)); csController->WaitCompactions(TDuration::Seconds(5)); } @@ -124,7 +120,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { WriteTestData(kikimr, "/Root/olapStore/olapTable", 30000, 1000000, 11000); while (csController->GetCompactionStartedCounter().Val() == 0) { - Cout << "Wait indexation..." << Endl; + Cout << "Wait compaction..." << Endl; Sleep(TDuration::Seconds(2)); } while (Singleton()->GetWritesCount() < 20 || @@ -133,7 +129,6 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { << csController->GetIndexWriteControllerBrokeCount().Val() << Endl; Sleep(TDuration::Seconds(2)); } - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); csController->WaitCompactions(TDuration::Seconds(5)); AFL_VERIFY(Singleton()->GetSize()); @@ -345,7 +340,7 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) { } while (csController->GetCompactionStartedCounter().Val() == 0) { - Cerr << "Wait indexation..." << Endl; + Cerr << "Wait compaction..." << Endl; Sleep(TDuration::Seconds(2)); } { diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index dbadc80b7454..5967585cb183 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -11405,13 +11405,8 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); } - void TestDropThenAddColumn(bool enableIndexation, bool enableCompaction) { - if (enableCompaction) { - Y_ABORT_UNLESS(enableIndexation); - } - + void TestDropThenAddColumn(bool enableCompaction) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); csController->DisableBackground(NYDBTest::ICSController::EBackground::Compaction); TKikimrSettings runnerSettings; @@ -11435,11 +11430,8 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } if (enableCompaction) { - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); csController->EnableBackground(NYDBTest::ICSController::EBackground::Compaction); - csController->WaitIndexation(TDuration::Seconds(5)); csController->WaitCompactions(TDuration::Seconds(5)); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); csController->DisableBackground(NYDBTest::ICSController::EBackground::Compaction); } @@ -11462,10 +11454,6 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { testHelper.BulkUpsert(testTable, tableInserter); } - if (enableIndexation) { - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); - } if (enableCompaction) { csController->EnableBackground(NYDBTest::ICSController::EBackground::Compaction); csController->WaitCompactions(TDuration::Seconds(5)); @@ -11475,15 +11463,11 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } Y_UNIT_TEST(DropThenAddColumn) { - TestDropThenAddColumn(false, false); - } - - Y_UNIT_TEST(DropThenAddColumnIndexation) { - TestDropThenAddColumn(true, true); + TestDropThenAddColumn(false); } Y_UNIT_TEST(DropThenAddColumnCompaction) { - TestDropThenAddColumn(true, true); + TestDropThenAddColumn(true); } Y_UNIT_TEST(DropTtlColumn) { @@ -11638,7 +11622,6 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { using namespace NArrow; auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -11665,18 +11648,12 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { auto alterAddResult = testHelper.GetSession().ExecuteSchemeQuery(alterQueryAdd).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterAddResult.GetStatus(), EStatus::SUCCESS, alterAddResult.GetIssues().ToString()); - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } - void TestInsertAddInsertDrop( - bool autoIndexation, bool indexationAfterInsertAddColumn, bool indexationAfterInsertDropColumn, bool indexationInEnd) { + void TestInsertAddInsertDrop() { using namespace NArrow; auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - if (!autoIndexation) { - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); - } TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -11704,42 +11681,22 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { auto alterAddResult = testHelper.GetSession().ExecuteSchemeQuery(alterQueryAdd).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterAddResult.GetStatus(), EStatus::SUCCESS, alterAddResult.GetIssues().ToString()); - if (!autoIndexation && indexationAfterInsertAddColumn) { - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); - } - testHelper.BulkUpsert(testTable, batch); auto alterQueryDrop = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "` DROP COLUMN column" << i << ";"; auto alterDropResult = testHelper.GetSession().ExecuteSchemeQuery(alterQueryDrop).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterDropResult.GetStatus(), EStatus::SUCCESS, alterDropResult.GetIssues().ToString()); - if (!autoIndexation && indexationAfterInsertDropColumn) { - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); - } - } - - if (!autoIndexation && indexationInEnd) { - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } } Y_UNIT_TEST(InsertAddInsertDrop) { - TestInsertAddInsertDrop(true, false, false, false); - for (i32 i = 0; i < 8; i++) { - TestInsertAddInsertDrop(false, i & 1, i & 2, i & 3); - } + TestInsertAddInsertDrop(); } Y_UNIT_TEST(DropTableAfterInsert) { using namespace NArrow; auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -11766,16 +11723,12 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { auto alterQueryDrop = TStringBuilder() << "DROP TABLE `" << testTable.GetName() << "`;"; auto alterDropResult = testHelper.GetSession().ExecuteSchemeQuery(alterQueryDrop).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterDropResult.GetStatus(), EStatus::SUCCESS, alterDropResult.GetIssues().ToString()); - - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } Y_UNIT_TEST(InsertDropAddColumn) { using namespace NArrow; auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - csController->DisableBackground(NYDBTest::ICSController::EBackground::Indexation); TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -11804,9 +11757,6 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { auto alterQueryAdd = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "` ADD COLUMN int_column Int32;"; auto alterAddResult = testHelper.GetSession().ExecuteSchemeQuery(alterQueryAdd).GetValueSync(); - - csController->EnableBackground(NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } Y_UNIT_TEST(CreateWithoutColumnFamily) { diff --git a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp index 96c721231978..9868b6739e65 100644 --- a/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp +++ b/ydb/core/kqp/ut/service/kqp_qs_queries_ut.cpp @@ -4510,7 +4510,6 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); const TString query = Sprintf(R"( CREATE TABLE `/Root/DataShard` ( @@ -4527,8 +4526,6 @@ Y_UNIT_TEST_SUITE(KqpQueryService) { auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); DoExecute(); - csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } }; diff --git a/ydb/core/kqp/ut/tx/kqp_sink_common.h b/ydb/core/kqp/ut/tx/kqp_sink_common.h index b29633af9b6f..08f2e564f2fe 100644 --- a/ydb/core/kqp/ut/tx/kqp_sink_common.h +++ b/ydb/core/kqp/ut/tx/kqp_sink_common.h @@ -39,7 +39,6 @@ class TTableDataModificationTester { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); csController->SetOverrideLagForCompactionBeforeTierings(TDuration::Seconds(1)); - csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); { auto type = IsOlap ? "COLUMN" : "ROW"; @@ -100,8 +99,6 @@ class TTableDataModificationTester { } DoExecute(); - csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); - csController->WaitIndexation(TDuration::Seconds(5)); } }; diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp index c054cc25bb33..efd5487f333b 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp deleted file mode 100644 index 15a05e7108a7..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "tx_gc_insert_table.h" -#include - -namespace NKikimr::NColumnShard { - -bool TTxInsertTableCleanup::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) { - TMemoryProfileGuard mpg("TTxInsertTableCleanup::Execute"); - TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - NIceDb::TNiceDb db(txc.DB); - - Self->TryAbortWrites(db, dbTable, std::move(WriteIdsToAbort)); - - NOlap::TBlobManagerDb blobManagerDb(txc.DB); - auto allAborted = Self->InsertTable->GetAborted(); - auto storage = Self->StoragesManager->GetInsertOperator(); - BlobsAction = storage->StartDeclareRemovingAction(NOlap::NBlobOperations::EConsumer::CLEANUP_INSERT_TABLE); - for (auto& [abortedWriteId, abortedData] : allAborted) { - Self->InsertTable->EraseAbortedOnExecute(dbTable, abortedData, BlobsAction); - } - BlobsAction->OnExecuteTxAfterRemoving(blobManagerDb, true); - return true; -} -void TTxInsertTableCleanup::Complete(const TActorContext& /*ctx*/) { - TMemoryProfileGuard mpg("TTxInsertTableCleanup::Complete"); - auto allAborted = Self->InsertTable->GetAborted(); - for (auto& [abortedWriteId, abortedData] : allAborted) { - Self->InsertTable->EraseAbortedOnComplete(abortedData); - } - - Y_ABORT_UNLESS(BlobsAction); - BlobsAction->OnCompleteTxAfterRemoving(true); - Self->BackgroundController.FinishCleanupInsertTable(); - Self->SetupCleanupInsertTable(); -} - -} diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.h b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.h deleted file mode 100644 index 96d8f09e5e6d..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once -#include -#include - -namespace NKikimr::NColumnShard { -class TTxInsertTableCleanup: public TTransactionBase { -private: - THashSet WriteIdsToAbort; - std::shared_ptr BlobsAction; -public: - TTxInsertTableCleanup(TColumnShard* self, THashSet&& writeIdsToAbort) - : TBase(self) - , WriteIdsToAbort(std::move(writeIdsToAbort)) { - Y_ABORT_UNLESS(WriteIdsToAbort.size() || self->InsertTable->GetAborted().size()); - } - - ~TTxInsertTableCleanup() { - } - - virtual bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; - virtual void Complete(const TActorContext& ctx) override; - TTxType GetTxType() const override { return TXTYPE_CLEANUP_INSERT_TABLE; } -}; - -} diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp deleted file mode 100644 index c25c44196197..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ /dev/null @@ -1,141 +0,0 @@ -#include "tx_write.h" - -#include -#include - -namespace NKikimr::NColumnShard { - -bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSerializedBatch& batch, const TInsertWriteId writeId) { - auto userData = batch.BuildInsertionUserData(*Self); - NOlap::TInsertedData insertData(writeId, userData); - - TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - bool ok = Self->InsertTable->Insert(dbTable, std::move(insertData)); - if (ok) { - Self->UpdateInsertTableCounters(); - return true; - } - return false; -} - -bool TTxWrite::CommitOneBlob(TTransactionContext& txc, const NOlap::TWideSerializedBatch& batch, const TInsertWriteId writeId) { - auto userData = batch.BuildInsertionUserData(*Self); - TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - AFL_VERIFY(CommitSnapshot); - NOlap::TCommittedData commitData(userData, *CommitSnapshot, Self->Generation(), writeId); - if (Self->TablesManager.HasTable(userData->GetPathId())) { - auto counters = Self->InsertTable->CommitEphemeral(dbTable, std::move(commitData)); - Self->Counters.GetTabletCounters()->OnWriteCommitted(counters); - } - Self->UpdateInsertTableCounters(); - return true; -} - -bool TTxWrite::DoExecute(TTransactionContext& txc, const TActorContext&) { - CommitSnapshot = Self->GetCurrentSnapshotForInternalModification(); - TMemoryProfileGuard mpg("TTxWrite::Execute"); - NActors::TLogContextGuard logGuard = - NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("tx_state", "execute"); - ACFL_DEBUG("event", "start_execute"); - const NOlap::TWritingBuffer& buffer = PutBlobResult->Get()->MutableWritesBuffer(); - const auto minReadSnapshot = Self->GetMinReadSnapshot(); - for (auto&& aggr : buffer.GetAggregations()) { - const auto& writeMeta = aggr->GetWriteMeta(); - Y_ABORT_UNLESS(Self->TablesManager.IsReadyForFinishWrite(writeMeta.GetPathId().InternalPathId, minReadSnapshot)); - txc.DB.NoMoreReadsForTx(); - TWriteOperation::TPtr operation; - AFL_VERIFY(!writeMeta.HasLongTxId()); - operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); - for (auto&& i : aggr->GetSplittedBlobs()) { - if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - static TAtomicCounter Counter = 0; - const TInsertWriteId insertWriteId = (TInsertWriteId)Counter.Inc(); - AFL_VERIFY(CommitOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( - "size", aggr->GetSplittedBlobs().size()); - } else { - const TInsertWriteId insertWriteId = Self->InsertTable->BuildNextWriteId(txc); - aggr->AddInsertWriteId(insertWriteId); - AFL_VERIFY(InsertOneBlob(txc, i, insertWriteId))("write_id", writeMeta.GetWriteId())("insert_write_id", insertWriteId)( - "size", aggr->GetSplittedBlobs().size()); - } - } - } - - NOlap::TBlobManagerDb blobManagerDb(txc.DB); - AFL_VERIFY(buffer.GetAddActions().size() == 1); - for (auto&& i : buffer.GetAddActions()) { - i->OnExecuteTxAfterWrite(*Self, blobManagerDb, true); - } - for (auto&& i : buffer.GetRemoveActions()) { - i->OnExecuteTxAfterRemoving(blobManagerDb, true); - } - Results.clear(); - for (auto&& aggr : buffer.GetAggregations()) { - const auto& writeMeta = aggr->GetWriteMeta(); - AFL_VERIFY(!writeMeta.HasLongTxId()); - auto operation = Self->OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - Y_ABORT_UNLESS(operation->GetStatus() == EOperationStatus::Started); - operation->OnWriteFinish(txc, aggr->GetInsertWriteIds(), operation->GetBehaviour() == EOperationBehaviour::NoTxWrite); - Self->OperationsManager->LinkInsertWriteIdToOperationWriteId(aggr->GetInsertWriteIds(), operation->GetWriteId()); - if (operation->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID()); - Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); - } else { - auto& info = Self->OperationsManager->GetLockVerified(operation->GetLockId()); - NKikimrDataEvents::TLock lock; - lock.SetLockId(operation->GetLockId()); - lock.SetDataShard(Self->TabletID()); - lock.SetGeneration(info.GetGeneration()); - lock.SetCounter(info.GetInternalGenerationCounter()); - writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(lock); - auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); - Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); - } - } - return true; -} - -void TTxWrite::DoComplete(const TActorContext& ctx) { - TMemoryProfileGuard mpg("TTxWrite::Complete"); - NActors::TLogContextGuard logGuard = - NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("tx_state", "complete"); - const auto now = TMonotonic::Now(); - const NOlap::TWritingBuffer& buffer = PutBlobResult->Get()->MutableWritesBuffer(); - for (auto&& i : buffer.GetAddActions()) { - i->OnCompleteTxAfterWrite(*Self, true); - } - for (auto&& i : buffer.GetRemoveActions()) { - i->OnCompleteTxAfterRemoving(true); - } - - AFL_VERIFY(buffer.GetAggregations().size() == Results.size()); - for (auto&& i : Results) { - i.DoSendReply(ctx); - } - for (ui32 i = 0; i < buffer.GetAggregations().size(); ++i) { - const auto& writeMeta = buffer.GetAggregations()[i]->GetWriteMeta(); - AFL_VERIFY(!writeMeta.HasLongTxId()); - auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - if (op->GetBehaviour() == EOperationBehaviour::WriteWithLock || op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - if (op->GetBehaviour() != EOperationBehaviour::NoTxWrite || Self->GetOperationsManager().HasReadLocks(writeMeta.GetPathId().InternalPathId)) { - auto evWrite = std::make_shared(writeMeta.GetPathId().InternalPathId, - buffer.GetAggregations()[i]->GetRecordBatch(), Self->GetIndexOptional()->GetVersionedIndex().GetPrimaryKey()); - Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite); - } - } - if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) { - Self->OperationsManager->AddTemporaryTxLink(op->GetLockId()); - AFL_VERIFY(CommitSnapshot); - Self->OperationsManager->CommitTransactionOnComplete(*Self, op->GetLockId(), *CommitSnapshot); - Self->Counters.GetTabletCounters()->IncCounter(COUNTER_IMMEDIATE_TX_COMPLETED); - } - Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant()); - Self->Counters.GetCSCounters().OnSuccessWriteResponse(); - } - Self->SetupIndexation(); -} - -} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.h b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.h deleted file mode 100644 index 6482947d0095..000000000000 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.h +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once -#include -#include -#include - -namespace NKikimr::NColumnShard { - -class TTxWrite: public TExtendedTransactionBase { -private: - using TBase = TExtendedTransactionBase; - -public: - TTxWrite(TColumnShard* self, const TEvPrivate::TEvWriteBlobsResult::TPtr& putBlobResult) - : TBase(self, "TTxWrite") - , PutBlobResult(putBlobResult) { - } - - bool DoExecute(TTransactionContext& txc, const TActorContext& ctx) override; - void DoComplete(const TActorContext& ctx) override; - TTxType GetTxType() const override { - return TXTYPE_WRITE; - } - -private: - TEvPrivate::TEvWriteBlobsResult::TPtr PutBlobResult; - std::optional CommitSnapshot; - - bool CommitOneBlob(TTransactionContext& txc, const NOlap::TWideSerializedBatch& batch, const TInsertWriteId writeId); - bool InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSerializedBatch& batch, const TInsertWriteId writeId); - - class TReplyInfo { - private: - std::unique_ptr Event; - TActorId DestinationForReply; - const ui64 Cookie; - - public: - TReplyInfo(std::unique_ptr&& ev, const TActorId& destinationForReply, const ui64 cookie) - : Event(std::move(ev)) - , DestinationForReply(destinationForReply) - , Cookie(cookie) { - } - - void DoSendReply(const TActorContext& ctx) { - ctx.Send(DestinationForReply, Event.release(), 0, Cookie); - } - }; - - std::vector Results; -}; - -} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index f996b91d242a..9719e5854583 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -10,7 +10,6 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) auto changes = Ev->Get()->IndexChanges; TMemoryProfileGuard mpg("TTxWriteIndex::Execute::" + changes->TypeString()); TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("external_task_id", changes->GetTaskIdentifier()); - Y_ABORT_UNLESS(Self->InsertTable); Y_ABORT_UNLESS(Self->TablesManager.HasPrimaryIndex()); txc.DB.NoMoreReadsForTx(); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/ya.make b/ydb/core/tx/columnshard/blobs_action/transaction/ya.make index 405f0e3f9ebd..ddafcc9cf302 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/ya.make +++ b/ydb/core/tx/columnshard/blobs_action/transaction/ya.make @@ -2,9 +2,7 @@ LIBRARY() SRCS( tx_draft.cpp - tx_write.cpp tx_write_index.cpp - tx_gc_insert_table.cpp tx_gc_indexed.cpp tx_remove_blobs.cpp tx_blobs_written.cpp diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index daa6ce49192b..b0f60d2d6e2c 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -289,19 +289,6 @@ void TColumnShard::Handle(TEvMediatorTimecast::TEvNotifyPlanStep::TPtr& ev, cons EnqueueBackgroundActivities(true); } -void TColumnShard::UpdateInsertTableCounters() { - auto& prepared = InsertTable->GetCountersPrepared(); - auto& committed = InsertTable->GetCountersCommitted(); - - Counters.GetTabletCounters()->SetCounter(COUNTER_PREPARED_RECORDS, prepared.Rows); - Counters.GetTabletCounters()->SetCounter(COUNTER_PREPARED_BYTES, prepared.Bytes); - Counters.GetTabletCounters()->SetCounter(COUNTER_COMMITTED_RECORDS, committed.Rows); - Counters.GetTabletCounters()->SetCounter(COUNTER_COMMITTED_BYTES, committed.Bytes); - - LOG_S_TRACE("InsertTable. Prepared: " << prepared.Bytes << " in " << prepared.Rows << " records, committed: " << committed.Bytes << " in " - << committed.Rows << " records at tablet " << TabletID()); -} - void TColumnShard::UpdateIndexCounters() { if (!TablesManager.HasPrimaryIndex()) { return; @@ -358,11 +345,7 @@ void TColumnShard::UpdateIndexCounters() { ui64 TColumnShard::MemoryUsage() const { ui64 memory = ProgressTxController->GetMemoryUsage() + ScanTxInFlight.size() * (sizeof(ui64) + sizeof(TInstant)) + - LongTxWrites.size() * (sizeof(TInsertWriteId) + sizeof(TLongTxWriteInfo)) + - LongTxWritesByUniqueId.size() * (sizeof(TULID) + sizeof(void*)) + - (WaitingScans.size()) * (sizeof(NOlap::TSnapshot) + sizeof(void*)) + - Counters.GetTabletCounters()->GetValue(COUNTER_PREPARED_RECORDS) * sizeof(NOlap::TInsertedData) + - Counters.GetTabletCounters()->GetValue(COUNTER_COMMITTED_RECORDS) * sizeof(NOlap::TInsertedData); + (WaitingScans.size()) * (sizeof(NOlap::TSnapshot) + sizeof(void*)); memory += TablesManager.GetMemoryUsage(); return memory; } diff --git a/ydb/core/tx/columnshard/columnshard.h b/ydb/core/tx/columnshard/columnshard.h index fefa2c5a62c1..641500233b2f 100644 --- a/ydb/core/tx/columnshard/columnshard.h +++ b/ydb/core/tx/columnshard/columnshard.h @@ -238,57 +238,6 @@ namespace TEvColumnShard { Record.SetTxId(txId); } }; - - struct TEvWrite : public TEventPB { - TEvWrite() = default; - - TEvWrite(const TActorId& source, const NLongTxService::TLongTxId& longTxId, NColumnShard::TSchemeShardLocalPathId tableId, - const TString& dedupId, const TString& data, const ui32 writePartId, - const NEvWrite::EModificationType modificationType) { - ActorIdToProto(source, Record.MutableSource()); - tableId.ToProto(Record); - Record.SetDedupId(dedupId); - Record.SetData(data); - Record.SetWritePartId(writePartId); - Record.SetModificationType(TEnumOperator::SerializeToProto(modificationType)); - longTxId.ToProto(Record.MutableLongTxId()); - } - - // Optionally set schema to deserialize data with - void SetArrowSchema(const TString& arrowSchema) { - Record.MutableMeta()->SetFormat(NKikimrTxColumnShard::FORMAT_ARROW); - Record.MutableMeta()->SetSchema(arrowSchema); - } - - void SetArrowData(const TString& arrowSchema, const TString& arrowData) { - Record.MutableMeta()->SetFormat(NKikimrTxColumnShard::FORMAT_ARROW); - Record.MutableMeta()->SetSchema(arrowSchema); - Record.SetData(arrowData); - } - }; - - struct TEvWriteResult : public TEventPB { - TEvWriteResult() = default; - - TEvWriteResult(ui64 origin, const NEvWrite::TWriteMeta& writeMeta, ui32 status) - : TEvWriteResult(origin, writeMeta, writeMeta.GetWriteId(), status) - { - } - - TEvWriteResult(ui64 origin, const NEvWrite::TWriteMeta& writeMeta, const i64 writeId, ui32 status) { - Record.SetOrigin(origin); - Record.SetTxInitiator(0); - Record.SetWriteId(writeId); - writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(Record); - Record.SetDedupId(writeMeta.GetDedupId()); - Record.SetStatus(status); - } - - Ydb::StatusIds::StatusCode GetYdbStatus() const { - const auto status = (NKikimrTxColumnShard::EResultStatus)Record.GetStatus(); - return NColumnShard::ConvertToYdbStatus(status); - } - }; }; inline auto& Proto(TEvColumnShard::TEvProposeTransaction* ev) { diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 9190c7c7ba77..a87bc0ae2c7c 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -52,19 +52,15 @@ void TTxInit::SetDefaults() { Self->LastCompletedTx = NOlap::TSnapshot::Zero(); Self->OwnerPathId = 0; Self->OwnerPath.clear(); - Self->LongTxWrites.clear(); - Self->LongTxWritesByUniqueId.clear(); } std::shared_ptr TTxInit::BuildReader() { auto result = std::make_shared("composite_init"); result->AddChildren(std::make_shared("special_values", Self)); result->AddChildren(std::make_shared("tables_manager", Self)); - result->AddChildren(std::make_shared("insert_table", Self)); result->AddChildren(std::make_shared("tx_controller", Self)); result->AddChildren(std::make_shared("operations_manager", Self)); result->AddChildren(std::make_shared("storages_manager", Self)); - result->AddChildren(std::make_shared("long_tx", Self)); result->AddChildren(std::make_shared("db_locks", Self)); result->AddChildren(std::make_shared("bg_sessions", Self)); result->AddChildren(std::make_shared("sharing_sessions", Self)); @@ -87,7 +83,6 @@ bool TTxInit::Execute(TTransactionContext& txc, const TActorContext& ctx) { return false; } StartReader = nullptr; - Self->UpdateInsertTableCounters(); Self->UpdateIndexCounters(); Self->UpdateResourceMetrics(ctx, {}); } catch (const TNotReadyTabletException&) { diff --git a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp index 44bb0a27f860..8156959e8903 100644 --- a/ydb/core/tx/columnshard/columnshard__progress_tx.cpp +++ b/ydb/core/tx/columnshard/columnshard__progress_tx.cpp @@ -112,7 +112,6 @@ class TColumnShard::TTxProgressTx: public TTransactionBase { Self->GetProgressTxController().GetCounters().OnTxExecuteDuration(TxOperator->GetOpType(), TMonotonic::Now() - *StartExecution); Self->GetProgressTxController().GetCounters().OnTxLiveDuration(TxOperator->GetOpType(), TMonotonic::Now() - ConstructionInstant); } - Self->SetupIndexation(); } }; diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 014ad139c9c4..c67aabd81139 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -2,7 +2,6 @@ #include "blobs_action/transaction/tx_blobs_written.h" #include "blobs_action/transaction/tx_draft.h" -#include "blobs_action/transaction/tx_write.h" #include "common/limits.h" #include "counters/columnshard.h" #include "engines/column_engine_logs.h" @@ -29,9 +28,6 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const case EOverloadStatus::Disk: Counters.OnWriteOverloadDisk(); break; - case EOverloadStatus::InsertTable: - Counters.OnWriteOverloadInsertTable(writeSize); - break; case EOverloadStatus::OverloadMetadata: Counters.OnWriteOverloadMetadata(writeSize); break; @@ -58,9 +54,6 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const } TColumnShard::EOverloadStatus TColumnShard::CheckOverloadedWait(const TInternalPathId pathId) const { - if (InsertTable && InsertTable->IsOverloadedByCommitted(pathId)) { - return EOverloadStatus::InsertTable; - } Counters.GetCSCounters().OnIndexMetadataLimit(NOlap::IColumnEngine::GetMetadataLimit()); if (TablesManager.GetPrimaryIndex()) { if (TablesManager.GetPrimaryIndex()->IsOverloadedByMetadata(NOlap::IColumnEngine::GetMetadataLimit())) { @@ -136,6 +129,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletID())("event", "TEvWriteBlobsResult"); auto& putResult = ev->Get()->GetPutResult(); + AFL_VERIFY(putResult.GetPutStatus() != NKikimrProto::OK); OnYellowChannels(putResult); NOlap::TWritingBuffer& wBuffer = ev->Get()->MutableWritesBuffer(); auto baseAggregations = wBuffer.GetAggregations(); @@ -148,26 +142,18 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo "writing_id", writeMeta.GetId())("status", putResult.GetPutStatus()); Counters.GetWritesMonitor()->OnFinishWrite(aggr->GetSize(), 1); - if (putResult.GetPutStatus() != NKikimrProto::OK) { - Counters.GetCSCounters().OnWritePutBlobsFail(TMonotonic::Now() - writeMeta.GetWriteStartInstant()); - Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL); - - AFL_VERIFY(!writeMeta.HasLongTxId()); - auto operation = OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); - auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), operation->GetLockId(), - ev->Get()->GetWriteResultStatus(), ev->Get()->GetErrorMessage() ? ev->Get()->GetErrorMessage() : "put data fails"); - ctx.Send(writeMeta.GetSource(), result.release(), 0, operation->GetCookie()); - Counters.GetCSCounters().OnFailedWriteResponse(EWriteFailReason::PutBlob); - wBuffer.RemoveData(aggr, StoragesManager->GetInsertOperator()); - } else { - const TMonotonic now = TMonotonic::Now(); - Counters.OnWritePutBlobsSuccess(now - writeMeta.GetWriteStartInstant(), aggr->GetRows()); - LOG_S_DEBUG("Write (record) into pathId " << writeMeta.GetPathId() - << (writeMeta.GetWriteId() ? (" writeId " + ToString(writeMeta.GetWriteId())).c_str() : "") - << " at tablet " << TabletID()); - } + Counters.GetCSCounters().OnWritePutBlobsFail(TMonotonic::Now() - writeMeta.GetWriteStartInstant()); + Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL); + + AFL_VERIFY(!writeMeta.HasLongTxId()); + auto operation = OperationsManager->GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), operation->GetLockId(), ev->Get()->GetWriteResultStatus(), + ev->Get()->GetErrorMessage() ? ev->Get()->GetErrorMessage() : "put data fails"); + ctx.Send(writeMeta.GetSource(), result.release(), 0, operation->GetCookie()); + Counters.GetCSCounters().OnFailedWriteResponse(EWriteFailReason::PutBlob); + wBuffer.RemoveData(aggr, StoragesManager->GetInsertOperator()); } - Execute(new TTxWrite(this, ev), ctx); + AFL_VERIFY(wBuffer.IsEmpty()); } void TColumnShard::Handle(TEvPrivate::TEvWriteDraft::TPtr& ev, const TActorContext& ctx) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 8e34f5f7a0b5..53a1372b4c57 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -17,9 +17,7 @@ #include "bg_tasks/manager/manager.h" #include "blobs_action/storages_manager/manager.h" #include "blobs_action/transaction/tx_gc_indexed.h" -#include "blobs_action/transaction/tx_gc_insert_table.h" #include "blobs_action/transaction/tx_remove_blobs.h" -#include "blobs_action/transaction/tx_gc_insert_table.h" #include "blobs_action/transaction/tx_gc_indexed.h" #include "blobs_reader/actor.h" #include "bg_tasks/events/events.h" @@ -30,7 +28,6 @@ #include "engines/changes/cleanup_portions.h" #include "engines/changes/cleanup_tables.h" #include "engines/changes/general_compaction.h" -#include "engines/changes/indexation.h" #include "engines/changes/ttl.h" #include "hooks/abstract/abstract.h" #include "resource_subscriber/counters.h" @@ -90,8 +87,6 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) Counters.GetPortionIndexCounters(), info->TabletID) , Subscribers(std::make_shared(*this)) , PipeClientCache(NTabletPipe::CreateBoundedClientCache(new NTabletPipe::TBoundedClientCacheConfig(), GetPipeClientConfig())) - , InsertTable(std::make_unique()) - , InsertTaskSubscription(NOlap::TInsertColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) , CompactTaskSubscription(NOlap::TCompactColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) , TTLTaskSubscription(NOlap::TTTLColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) , BackgroundController(Counters.GetBackgroundControllerCounters()) @@ -213,96 +208,6 @@ NOlap::TSnapshot TColumnShard::GetMinReadSnapshot() const { return NOlap::TSnapshot::MaxForPlanStep(minReadStep); } -TInsertWriteId TColumnShard::HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const { - auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId); - if (it != LongTxWritesByUniqueId.end()) { - auto itPart = it->second.find(partId); - if (itPart != it->second.end()) { - return itPart->second->InsertWriteId; - } - } - return (TInsertWriteId)0; -} - -TInsertWriteId TColumnShard::GetLongTxWrite(NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional granuleShardingVersionId) { - auto it = LongTxWritesByUniqueId.find(longTxId.UniqueId); - if (it != LongTxWritesByUniqueId.end()) { - auto itPart = it->second.find(partId); - if (itPart != it->second.end()) { - return itPart->second->InsertWriteId; - } - } else { - it = LongTxWritesByUniqueId.emplace(longTxId.UniqueId, TPartsForLTXShard()).first; - } - - TInsertWriteId insertWriteId = InsertTable->BuildNextWriteId(db); - auto& lw = LongTxWrites[insertWriteId]; - lw.InsertWriteId = insertWriteId; - lw.WritePartId = partId; - lw.LongTxId = longTxId; - lw.GranuleShardingVersionId = granuleShardingVersionId; - it->second[partId] = &lw; - - Schema::SaveLongTxWrite(db, insertWriteId, partId, longTxId, granuleShardingVersionId); - return insertWriteId; -} - -void TColumnShard::AddLongTxWrite(const TInsertWriteId writeId, ui64 txId) { - auto it = LongTxWrites.find(writeId); - AFL_VERIFY(it != LongTxWrites.end()); - it->second.PreparedTxId = txId; -} - -void TColumnShard::LoadLongTxWrite(const TInsertWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, const std::optional granuleShardingVersion) { - auto& lw = LongTxWrites[writeId]; - lw.WritePartId = writePartId; - lw.InsertWriteId = writeId; - lw.LongTxId = longTxId; - lw.GranuleShardingVersionId = granuleShardingVersion; - LongTxWritesByUniqueId[longTxId.UniqueId][writePartId] = &lw; -} - -bool TColumnShard::RemoveLongTxWrite(NIceDb::TNiceDb& db, const TInsertWriteId writeId, const ui64 txId) { - if (auto* lw = LongTxWrites.FindPtr(writeId)) { - ui64 prepared = lw->PreparedTxId; - if (!prepared || txId == prepared) { - Schema::EraseLongTxWrite(db, writeId); - auto& ltxParts = LongTxWritesByUniqueId[lw->LongTxId.UniqueId]; - ltxParts.erase(lw->WritePartId); - if (ltxParts.empty()) { - LongTxWritesByUniqueId.erase(lw->LongTxId.UniqueId); - } - LongTxWrites.erase(writeId); - return true; - } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "cannot_remove_prepared_tx_insertion")("write_id", (ui64)writeId)("tx_id", txId); - return false; - } - } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "cannot_remove_removed_tx_insertion")("write_id", (ui64)writeId)("tx_id", txId); - return true; - } -} - -void TColumnShard::TryAbortWrites(NIceDb::TNiceDb& db, NOlap::TDbWrapper& dbTable, THashSet&& writesToAbort) { - std::vector failedAborts; - for (auto& writeId : writesToAbort) { - if (!RemoveLongTxWrite(db, writeId, 0)) { - failedAborts.push_back(writeId); - } - } - if (failedAborts.size()) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "failed_aborts")("count", failedAborts.size())("writes_count", writesToAbort.size()); - } - for (auto& writeId : failedAborts) { - InsertTable->MarkAsNotAbortable(writeId); - writesToAbort.erase(writeId); - } - if (!writesToAbort.empty()) { - InsertTable->Abort(dbTable, writesToAbort); - } -} - void TColumnShard::UpdateSchemaSeqNo(const TMessageSeqNo& seqNo, NTabletFlatExecutor::TTransactionContext& txc) { if (LastSchemaSeqNo < seqNo) { LastSchemaSeqNo = seqNo; @@ -432,7 +337,6 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl tableVerProto.SetSchemaPresetVersionAdj(tableProto.GetSchemaPresetVersionAdj()); TablesManager.AddTableVersion(internalPathId, version, tableVerProto, schema, db); - InsertTable->RegisterPathInfo(internalPathId); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLES, TablesManager.GetTables().size()); Counters.GetTabletCounters()->SetCounter(COUNTER_TABLE_PRESETS, TablesManager.GetSchemaPresets().size()); @@ -532,14 +436,12 @@ void TColumnShard::EnqueueBackgroundActivities(const bool periodic) { // !!!!!! MUST BE FIRST THROUGH DATA HAVE TO BE SAME IN SESSIONS AFTER TABLET RESTART SharingSessionsManager->Start(*this); - SetupIndexation(); SetupCompaction({}); SetupCleanupPortions(); SetupCleanupTables(); SetupMetadata(); SetupTtl(); SetupGC(); - SetupCleanupInsertTable(); } class TChangesTask: public NConveyor::ITask { @@ -599,15 +501,10 @@ class TChangesReadTask: public NOlap::NBlobOperations::NRead::ITask { AFL_VERIFY(TxEvent->IndexChanges->ResourcesGuard); } TxEvent->IndexChanges->Blobs = ExtractBlobsData(); - const bool isInsert = !!dynamic_pointer_cast(TxEvent->IndexChanges); TxEvent->IndexChanges->SetStage(NOlap::NChanges::EStage::ReadyForConstruct); std::shared_ptr task = std::make_shared(std::move(TxEvent), Counters, TabletId, ParentActorId, LastCompletedTx); - if (isInsert) { - NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); - } else { - NConveyorComposite::TCompServiceOperator::SendTaskToExecute(task); - } + NConveyorComposite::TCompServiceOperator::SendTaskToExecute(task); } virtual bool DoOnError(const TString& storageId, const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "DoOnError")("storage_id", storageId)("blob_id", range)("status", status.GetErrorMessage())("status_code", status.GetStatus()); @@ -715,14 +612,6 @@ class TDataAccessorsSubscriberWithRead: public TDataAccessorsSubscriber { } }; -class TInsertChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCounter { -private: - using TBase = TChangesReadTask; - -public: - using TBase::TBase; -}; - class TCompactChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCounter { private: using TBase = TChangesReadTask; @@ -739,85 +628,6 @@ class TTTLChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCou using TBase::TBase; }; -void TColumnShard::StartIndexTask(std::vector&& dataToIndex, const i64 bytesToIndex) { - Counters.GetCSCounters().IndexationInput(bytesToIndex); - - std::vector data; - data.reserve(dataToIndex.size()); - for (auto& ptr : dataToIndex) { - data.push_back(*ptr); - if (!TablesManager.HasTable(data.back().GetPathId())) { - data.back().SetRemove(); - } - } - - Y_ABORT_UNLESS(data.size()); - auto indexChanges = TablesManager.MutablePrimaryIndex().StartInsert(std::move(data)); - Y_ABORT_UNLESS(indexChanges); - - auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); - indexChanges->Start(*this); - auto ev = std::make_unique(actualIndexInfo, indexChanges, Settings.CacheDataAfterIndexing); - - const TString externalTaskId = indexChanges->GetTaskIdentifier(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "indexation")("bytes", bytesToIndex)("blobs_count", dataToIndex.size())("max_limit", (i64)Limits.MaxInsertBytes)("has_more", bytesToIndex >= Limits.MaxInsertBytes)("external_task_id", externalTaskId); - - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared( - std::make_shared(std::move(ev), SelfId(), TabletID(), Counters.GetIndexationCounters(), GetLastCompletedTx()), - 0, indexChanges->CalcMemoryForUsage(), externalTaskId, InsertTaskSubscription)); -} - -void TColumnShard::SetupIndexation() { - if (!AppDataVerified().ColumnShardConfig.GetIndexationEnabled() || !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Indexation)) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "disabled"); - return; - } - BackgroundController.CheckDeadlinesIndexation(); - if (BackgroundController.GetIndexingActiveCount()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "in_progress")("count", BackgroundController.GetIndexingActiveCount())("insert_overload_size", InsertTable->GetCountersCommitted().Bytes)("indexing_debug", BackgroundController.DebugStringIndexation()); - return; - } - - bool force = false; - if (InsertTable->GetPathPriorities().size() && InsertTable->GetPathPriorities().rbegin()->first.GetCategory() == NOlap::TPathInfoIndexPriority::EIndexationPriority::PreventOverload) { - force = true; - } - const ui64 bytesLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationStartBytesLimit(); - const TDuration durationLimit = NYDBTest::TControllers::GetColumnShardController()->GetGuaranteeIndexationInterval(); - if (!force && InsertTable->GetCountersCommitted().Bytes < bytesLimit && - TMonotonic::Now() < BackgroundController.GetLastIndexationInstant() + durationLimit) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_indexation")("reason", "not_enough_data_and_too_frequency")("insert_size", InsertTable->GetCountersCommitted().Bytes); - return; - } - - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "start_indexation_tasks")("insert_overload_size", InsertTable->GetCountersCommitted().Bytes); - Counters.GetCSCounters().OnSetupIndexation(); - ui64 bytesToIndex = 0; - ui64 txBytesWrite = 0; - std::vector dataToIndex; - dataToIndex.reserve(TLimits::MIN_SMALL_BLOBS_TO_INSERT); - for (auto it = InsertTable->GetPathPriorities().rbegin(); it != InsertTable->GetPathPriorities().rend(); ++it) { - for (auto* pathInfo : it->second) { - for (auto& data : pathInfo->GetCommitted()) { - Y_ABORT_UNLESS(data.BlobSize()); - bytesToIndex += data.BlobSize(); - txBytesWrite += data.GetTxVolume(); - dataToIndex.push_back(&data); - if (bytesToIndex >= (ui64)Limits.MaxInsertBytes || txBytesWrite >= NOlap::TGlobalLimits::TxWriteLimitBytes || dataToIndex.size() > 500) { - StartIndexTask(std::move(dataToIndex), bytesToIndex); - dataToIndex.clear(); - bytesToIndex = 0; - txBytesWrite = 0; - } - } - } - } - if (dataToIndex.size()) { - StartIndexTask(std::move(dataToIndex), bytesToIndex); - } -} - namespace { class TCompactionAllocated: public NPrioritiesQueue::IRequest { private: @@ -1100,9 +910,6 @@ void TColumnShard::SetupCleanupTables() { THashSet pathIdsEmptyInInsertTable; for (auto&& i : TablesManager.GetPathsToDrop(GetMinReadSnapshot())) { - if (InsertTable->HasPathIdData(i)) { - continue; - } pathIdsEmptyInInsertTable.emplace(i); } @@ -1151,22 +958,6 @@ void TColumnShard::Handle(TEvPrivate::TEvGarbageCollectionFinished::TPtr& ev, co Execute(new TTxGarbageCollectionFinished(this, ev->Get()->Action), ctx); } -void TColumnShard::SetupCleanupInsertTable() { - auto writeIdsToCleanup = InsertTable->OldWritesToAbort(AppData()->TimeProvider->Now()); - - if (BackgroundController.IsCleanupInsertTableActive()) { - ACFL_DEBUG("background", "cleanup_insert_table")("skip_reason", "in_progress"); - return; - } - - if (!InsertTable->GetAborted().size() && !writeIdsToCleanup.size()) { - return; - } - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "cleanup_started")("aborted", InsertTable->GetAborted().size())("to_cleanup", writeIdsToCleanup.size()); - BackgroundController.StartCleanupInsertTable(); - Execute(new TTxInsertTableCleanup(this, std::move(writeIdsToCleanup)), TActorContext::AsActorContext()); -} - void TColumnShard::Die(const TActorContext& ctx) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "tablet_die"); AFL_VERIFY(TabletActivityImpl->Dec() == 0); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 73de964f7deb..0dcda3a5b224 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -99,7 +99,6 @@ class TArrowData; class TEvWriteCommitPrimaryTransactionOperator; class TEvWriteCommitSecondaryTransactionOperator; class TTxFinishAsyncTransaction; -class TTxInsertTableCleanup; class TTxRemoveSharedBlobs; class TOperationsManager; class TWaitEraseTablesTxSubscriber; @@ -109,11 +108,9 @@ class TWriteTasksQueue; class TWriteTask; namespace NLoading { -class TInsertTableInitializer; class TTxControllerInitializer; class TOperationsManagerInitializer; class TStoragesManagerInitializer; -class TLongTxInitializer; class TDBLocksInitializer; class TBackgroundSessionsInitializer; class TSharingSessionsInitializer; @@ -162,7 +159,6 @@ using TTransactionBase = NTabletFlatExecutor::TTransactionBase; class TColumnShard: public TActor, public NTabletFlatExecutor::TTabletExecutedFlat { friend class TEvWriteCommitSecondaryTransactionOperator; friend class TEvWriteCommitPrimaryTransactionOperator; - friend class TTxInsertTableCleanup; friend class TTxInit; friend class TTxInitSchema; friend class TTxUpdateSchema; @@ -222,17 +218,14 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa friend class TWriteOperation; friend class TSchemaTransactionOperator; - friend class TLongTxTransactionOperator; friend class TEvWriteTransactionOperator; friend class TBackupTransactionOperator; friend class IProposeTxOperator; friend class TSharingTransactionOperator; - friend class NLoading::TInsertTableInitializer; friend class NLoading::TTxControllerInitializer; friend class NLoading::TOperationsManagerInitializer; friend class NLoading::TStoragesManagerInitializer; - friend class NLoading::TLongTxInitializer; friend class NLoading::TDBLocksInitializer; friend class NLoading::TBackgroundSessionsInitializer; friend class NLoading::TSharingSessionsInitializer; @@ -490,14 +483,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa std::optional StartInstant; bool IsTxInitFinished = false; - struct TLongTxWriteInfo { - TInsertWriteId InsertWriteId; - ui32 WritePartId; - NLongTxService::TLongTxId LongTxId; - ui64 PreparedTxId = 0; - std::optional GranuleShardingVersionId; - }; - ui64 CurrentSchemeShardId = 0; TMessageSeqNo LastSchemaSeqNo; std::optional ProcessingParams; @@ -532,16 +517,11 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa std::shared_ptr Subscribers; std::shared_ptr Tiers; std::unique_ptr PipeClientCache; - std::unique_ptr InsertTable; - NOlap::NResourceBroker::NSubscribe::TTaskContext InsertTaskSubscription; NOlap::NResourceBroker::NSubscribe::TTaskContext CompactTaskSubscription; NOlap::NResourceBroker::NSubscribe::TTaskContext TTLTaskSubscription; std::optional ProgressTxInFlight; THashMap ScanTxInFlight; - THashMap LongTxWrites; - using TPartsForLTXShard = THashMap; - THashMap LongTxWritesByUniqueId; TMultiMap WaitingScans; TBackgroundController BackgroundController; TSettings Settings; @@ -553,7 +533,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void TryRegisterMediatorTimeCast(); void UnregisterMediatorTimeCast(); - void TryAbortWrites(NIceDb::TNiceDb& db, NOlap::TDbWrapper& dbTable, THashSet&& writesToAbort); bool WaitPlanStep(ui64 step); void SendWaitPlanStep(ui64 step); @@ -566,14 +545,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa return ProgressTxController->GetTxCompleteLag(mediatorTime); } - TInsertWriteId HasLongTxWrite(const NLongTxService::TLongTxId& longTxId, const ui32 partId) const; - TInsertWriteId GetLongTxWrite( - NIceDb::TNiceDb& db, const NLongTxService::TLongTxId& longTxId, const ui32 partId, const std::optional granuleShardingVersionId); - void AddLongTxWrite(const TInsertWriteId writeId, ui64 txId); - void LoadLongTxWrite(const TInsertWriteId writeId, const ui32 writePartId, const NLongTxService::TLongTxId& longTxId, - const std::optional granuleShardingVersion); - bool RemoveLongTxWrite(NIceDb::TNiceDb& db, const TInsertWriteId writeId, const ui64 txId); - void EnqueueBackgroundActivities(const bool periodic = false); virtual void Enqueue(STFUNC_SIG) override; @@ -592,8 +563,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void RunAlterStore( const NKikimrTxColumnShard::TAlterStore& body, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc); - void StartIndexTask(std::vector&& dataToIndex, const i64 bytesToIndex); - void SetupIndexation(); void SetupCompaction(const std::set& pathIds); void StartCompaction(const std::shared_ptr& guard); @@ -601,10 +570,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa bool SetupTtl(); void SetupCleanupPortions(); void SetupCleanupTables(); - void SetupCleanupInsertTable(); void SetupGC(); - void UpdateInsertTableCounters(); void UpdateIndexCounters(); void UpdateResourceMetrics(const TActorContext& ctx, const TUsage& usage); ui64 MemoryUsage() const; @@ -624,9 +591,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa return TablesManager; } - bool HasLongTxWrites(const TInsertWriteId insertWriteId) const { - return LongTxWrites.contains(insertWriteId); - } void EnqueueProgressTx(const TActorContext& ctx, const std::optional continueTxId); NOlap::TSnapshot GetLastTxSnapshot() const { return NOlap::TSnapshot(LastPlannedStep, LastPlannedTxId); @@ -690,11 +654,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa return DataLocksManager; } - const NOlap::TInsertTable& GetInsertTable() const { - AFL_VERIFY(!!InsertTable); - return *InsertTable; - } - static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::TX_COLUMNSHARD_ACTOR; } diff --git a/ydb/core/tx/columnshard/columnshard_schema.cpp b/ydb/core/tx/columnshard/columnshard_schema.cpp index c9a60029a892..6bdd0ab1053f 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/columnshard_schema.cpp @@ -3,34 +3,6 @@ namespace NKikimr::NColumnShard { -bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsGroupSelector, NOlap::TInsertTableAccessor& insertTable, const TInstant& /*loadTime*/) { - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - NOlap::TInsertTableRecordLoadContext constructor; - constructor.ParseFromDatabase(rowset); - - switch (constructor.GetRecType()) { - case Schema::EInsertTableIds::Inserted: - insertTable.AddInserted(constructor.BuildInsertedOrAborted(dsGroupSelector), true); - break; - case Schema::EInsertTableIds::Committed: - insertTable.AddCommitted(constructor.BuildCommitted(dsGroupSelector), true); - break; - case Schema::EInsertTableIds::Aborted: - insertTable.AddAborted(constructor.BuildInsertedOrAborted(dsGroupSelector), true); - break; - } - if (!rowset.Next()) { - return false; - } - } - return true; -} - void Schema::SaveTxInfo(NIceDb::TNiceDb& db, const TFullTxInfo& txInfo, const TString& txBody) { AFL_VERIFY(txInfo.TxKind != NKikimrTxColumnShard::TX_KIND_NONE); db.Table().Key(txInfo.TxId).Update( diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 6dba64d8f7e2..d0551ddccf44 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -26,8 +25,6 @@ struct Schema : NIceDb::Schema { using TSettings = SchemaSettings; - using TInsertedData = NOlap::TInsertedData; - using TCommittedData = NOlap::TCommittedData; using TColumnRecord = NOlap::TColumnRecord; enum EIndexTables : ui32 { @@ -837,70 +834,6 @@ struct Schema : NIceDb::Schema { db.Table().Key((ui64)writeId).Delete(); } - // InsertTable activities - - static void InsertTable_Upsert(NIceDb::TNiceDb& db, const EInsertTableIds recType, const TInsertedData& data) { - db.Table() - .Key((ui8)recType, 0, (ui64)data.GetInsertWriteId(), data.GetPathId().GetRawValue(), "") - .Update(NIceDb::TUpdate(data.GetBlobRange().GetBlobId().ToStringLegacy()), - NIceDb::TUpdate(data.GetBlobRange().Offset), - NIceDb::TUpdate((ui64)data.GetInsertWriteId()), - NIceDb::TUpdate(data.GetBlobRange().Size), - NIceDb::TUpdate(data.GetMeta().SerializeToProto().SerializeAsString()), - NIceDb::TUpdate(data.GetSchemaVersion())); - } - - static void InsertTable_Upsert(NIceDb::TNiceDb& db, const TCommittedData& data) { - db.Table() - .Key((ui8)EInsertTableIds::Committed, data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId().GetRawValue(), - data.GetDedupId()) - .Update(NIceDb::TUpdate(data.GetBlobRange().GetBlobId().ToStringLegacy()), - NIceDb::TUpdate((ui64)data.GetInsertWriteId()), - NIceDb::TUpdate(data.GetBlobRange().Offset), - NIceDb::TUpdate(data.GetBlobRange().Size), - NIceDb::TUpdate(data.GetMeta().SerializeToProto().SerializeAsString()), - NIceDb::TUpdate(data.GetSchemaVersion())); - } - - static void InsertTable_Erase(NIceDb::TNiceDb& db, EInsertTableIds recType, const TInsertedData& data) { - db.Table().Key((ui8)recType, 0, (ui64)data.GetInsertWriteId(), data.GetPathId().GetRawValue(), "").Delete(); - } - - static void InsertTable_Erase(NIceDb::TNiceDb& db, const TCommittedData& data) { - db.Table() - .Key((ui8)EInsertTableIds::Committed, data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId().GetRawValue(), data.GetDedupId()) - .Delete(); - } - - static void InsertTable_Insert(NIceDb::TNiceDb& db, const TInsertedData& data) { - InsertTable_Upsert(db, EInsertTableIds::Inserted, data); - } - - static void InsertTable_Commit(NIceDb::TNiceDb& db, const TCommittedData& data) { - InsertTable_Upsert(db, data); - } - - static void InsertTable_Abort(NIceDb::TNiceDb& db, const TInsertedData& data) { - InsertTable_Upsert(db, EInsertTableIds::Aborted, data); - } - - static void InsertTable_EraseInserted(NIceDb::TNiceDb& db, const TInsertedData& data) { - InsertTable_Erase(db, EInsertTableIds::Inserted, data); - } - - static void InsertTable_EraseCommitted(NIceDb::TNiceDb& db, const TCommittedData& data) { - InsertTable_Erase(db, data); - } - - static void InsertTable_EraseAborted(NIceDb::TNiceDb& db, const TInsertedData& data) { - InsertTable_Erase(db, EInsertTableIds::Aborted, data); - } - - static bool InsertTable_Load(NIceDb::TNiceDb& db, - const IBlobGroupSelector* dsGroupSelector, - NOlap::TInsertTableAccessor& insertTable, - const TInstant& loadTime); - // IndexCounters static void IndexCounters_Write(NIceDb::TNiceDb& db, ui32 counterId, ui64 value) { @@ -1165,122 +1098,4 @@ class TIndexChunkLoadContext { } }; -class TInsertTableRecordLoadContext { -private: - NColumnShard::Schema::EInsertTableIds RecType; - ui64 PlanStep; - ui64 WriteTxId; - TInsertWriteId InsertWriteId; - TInternalPathId PathId; - YDB_ACCESSOR_DEF(TString, DedupId); - ui64 SchemaVersion; - TString BlobIdString; - std::optional BlobId; - TString MetadataString; - std::optional Metadata; - ui64 RangeOffset; - ui64 RangeSize; - - void Prepare(const IBlobGroupSelector* dsGroupSelector) { - AFL_VERIFY(!PreparedFlag); - PreparedFlag = true; - TString error; - NOlap::TUnifiedBlobId blobId = NOlap::TUnifiedBlobId::ParseFromString(BlobIdString, dsGroupSelector, error); - Y_ABORT_UNLESS(blobId.IsValid(), "Failied to parse blob id: %s", error.c_str()); - BlobId = blobId; - - NKikimrTxColumnShard::TLogicalMetadata meta; - AFL_VERIFY(MetadataString); - Y_ABORT_UNLESS(meta.ParseFromString(MetadataString)); - Metadata = std::move(meta); - } - - bool PreparedFlag = false; - bool ParsedFlag = false; - -public: - TInsertWriteId GetInsertWriteId() const { - AFL_VERIFY(ParsedFlag); - return InsertWriteId; - } - - ui64 GetTxId() const { - AFL_VERIFY(ParsedFlag); - AFL_VERIFY(RecType == NColumnShard::Schema::EInsertTableIds::Committed); - return WriteTxId; - } - - NColumnShard::Schema::EInsertTableIds GetRecType() const { - AFL_VERIFY(ParsedFlag); - return RecType; - } - - ui64 GetPlanStep() const { - AFL_VERIFY(ParsedFlag); - AFL_VERIFY(RecType == NColumnShard::Schema::EInsertTableIds::Committed); - return PlanStep; - } - - void Remove(NIceDb::TNiceDb& db) const { - AFL_VERIFY(ParsedFlag); - db.Table().Key((ui8)RecType, PlanStep, WriteTxId, PathId.GetRawValue(), DedupId).Delete(); - } - - void Upsert(NIceDb::TNiceDb& db) const { - AFL_VERIFY(ParsedFlag); - using namespace NColumnShard; - db.Table() - .Key((ui8)RecType, PlanStep, WriteTxId, PathId.GetRawValue(), DedupId) - .Update(NIceDb::TUpdate(BlobIdString), - NIceDb::TUpdate(RangeOffset), - NIceDb::TUpdate(RangeSize), NIceDb::TUpdate(MetadataString), - NIceDb::TUpdate(SchemaVersion)); - } - - template - void ParseFromDatabase(TRowset& rowset) { - AFL_VERIFY(!ParsedFlag)("problem", "duplication parsing"); - ParsedFlag = true; - using namespace NColumnShard; - RecType = (Schema::EInsertTableIds)rowset.template GetValue(); - PlanStep = rowset.template GetValue(); - WriteTxId = rowset.template GetValueOrDefault(); - AFL_VERIFY(WriteTxId); - InsertWriteId = (TInsertWriteId)rowset.template GetValueOrDefault(WriteTxId); - - PathId = TInternalPathId::FromRawValue(rowset.template GetValue()); - DedupId = rowset.template GetValue(); - SchemaVersion = rowset.template GetValueOrDefault(0); - BlobIdString = rowset.template GetValue(); - MetadataString = rowset.template GetValue(); - AFL_VERIFY(rowset.template HaveValue()); - AFL_VERIFY(rowset.template HaveValue()); - RangeOffset = rowset.template GetValue(); - RangeSize = rowset.template GetValue(); - } - - NOlap::TCommittedData BuildCommitted(const IBlobGroupSelector* dsGroupSelector) { - Prepare(dsGroupSelector); - using namespace NColumnShard; - AFL_VERIFY(RecType == Schema::EInsertTableIds::Committed); - auto userData = std::make_shared( - PathId, NOlap::TBlobRange(*BlobId, RangeOffset, RangeSize), *Metadata, SchemaVersion, std::nullopt); - AFL_VERIFY(!!DedupId); - AFL_VERIFY(PlanStep); - return NOlap::TCommittedData(userData, PlanStep, WriteTxId, InsertWriteId, DedupId); - } - - NOlap::TInsertedData BuildInsertedOrAborted(const IBlobGroupSelector* dsGroupSelector) { - Prepare(dsGroupSelector); - using namespace NColumnShard; - AFL_VERIFY(InsertWriteId == (TInsertWriteId)WriteTxId)("insert", InsertWriteId)("write", WriteTxId); - AFL_VERIFY(RecType != Schema::EInsertTableIds::Committed); - auto userData = std::make_shared( - PathId, NOlap::TBlobRange(*BlobId, RangeOffset, RangeSize), *Metadata, SchemaVersion, std::nullopt); - AFL_VERIFY(!DedupId); - AFL_VERIFY(!PlanStep); - return NOlap::TInsertedData(InsertWriteId, userData); - } -}; - } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/counters/columnshard.cpp b/ydb/core/tx/columnshard/counters/columnshard.cpp index edbee47654b1..e2a2905ae72f 100644 --- a/ydb/core/tx/columnshard/counters/columnshard.cpp +++ b/ydb/core/tx/columnshard/counters/columnshard.cpp @@ -33,8 +33,6 @@ TCSCounters::TCSCounters() IndexMetadataLimitBytes = TBase::GetValue("IndexMetadata/Limit/Bytes"); - OverloadInsertTableBytes = TBase::GetDeriviative("Overload/InsertTable/Bytes"); - OverloadInsertTableCount = TBase::GetDeriviative("Overload/InsertTable/Count"); OverloadMetadataBytes = TBase::GetDeriviative("Overload/Metadata/Bytes"); OverloadMetadataCount = TBase::GetDeriviative("Overload/Metadata/Count"); OverloadCompactionBytes = TBase::GetDeriviative("Overload/Compaction/Bytes"); diff --git a/ydb/core/tx/columnshard/counters/columnshard.h b/ydb/core/tx/columnshard/counters/columnshard.h index d51085a40286..0716e21d4314 100644 --- a/ydb/core/tx/columnshard/counters/columnshard.h +++ b/ydb/core/tx/columnshard/counters/columnshard.h @@ -15,7 +15,6 @@ enum class EOverloadStatus { ShardTxInFly /* "shard_tx" */ = 0, ShardWritesInFly /* "shard_writes" */, ShardWritesSizeInFly /* "shard_writes_size" */, - InsertTable /* "insert_table" */, OverloadMetadata /* "overload_metadata" */, Disk /* "disk" */, None /* "none" */, @@ -94,8 +93,6 @@ class TCSCounters: public TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr IndexMetadataLimitBytes; - NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableBytes; - NMonitoring::TDynamicCounters::TCounterPtr OverloadInsertTableCount; NMonitoring::TDynamicCounters::TCounterPtr OverloadMetadataBytes; NMonitoring::TDynamicCounters::TCounterPtr OverloadMetadataCount; NMonitoring::TDynamicCounters::TCounterPtr OverloadCompactionBytes; @@ -197,11 +194,6 @@ class TCSCounters: public TCommonCountersOwner { SplitCompactionGranulePortionsCount->SetValue(portionsCount); } - void OnWriteOverloadInsertTable(const ui64 size) const { - OverloadInsertTableBytes->Add(size); - OverloadInsertTableCount->Add(1); - } - void OnWriteOverloadMetadata(const ui64 size) const { OverloadMetadataBytes->Add(size); OverloadMetadataCount->Add(1); diff --git a/ydb/core/tx/columnshard/counters/counters_manager.h b/ydb/core/tx/columnshard/counters/counters_manager.h index 2ade7ca6770a..8cf93aa0caaf 100644 --- a/ydb/core/tx/columnshard/counters/counters_manager.h +++ b/ydb/core/tx/columnshard/counters/counters_manager.h @@ -57,11 +57,6 @@ class TCountersManager { TabletCounters->IncCounter(COUNTER_OUT_OF_SPACE); } - void OnWriteOverloadInsertTable(const ui64 size) const { - TabletCounters->IncCounter(COUNTER_WRITE_OVERLOAD); - CSCounters.OnWriteOverloadInsertTable(size); - } - void OnWriteOverloadMetadata(const ui64 size) const { TabletCounters->IncCounter(COUNTER_WRITE_OVERLOAD); CSCounters.OnWriteOverloadMetadata(size); diff --git a/ydb/core/tx/columnshard/counters/insert_table.cpp b/ydb/core/tx/columnshard/counters/insert_table.cpp deleted file mode 100644 index 4d9e9d9f734e..000000000000 --- a/ydb/core/tx/columnshard/counters/insert_table.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "insert_table.h" - -namespace NKikimr::NColumnShard { - -TInsertTableCounters::TInsertTableCounters() - : TBase("InsertTable") - , PathIdInserted("InsertTable", "Inserted") - , PathIdCommitted("InsertTable", "Committed") - , Inserted("InsertTable", "Inserted") - , Committed("InsertTable", "Committed") - , Aborted("InsertTable", "Aborted") - , LoadCounters(NColumnShard::TLoadTimeSignals::TSignalsRegistry::GetSignal("InsertTable")) -{ -} - -} diff --git a/ydb/core/tx/columnshard/counters/insert_table.h b/ydb/core/tx/columnshard/counters/insert_table.h deleted file mode 100644 index 32432acc164f..000000000000 --- a/ydb/core/tx/columnshard/counters/insert_table.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once -#include "common_data.h" -#include -#include - -namespace NKikimr::NColumnShard { - -class TPathIdClientCounters { -private: - using TBase = TCommonCountersOwner; - const std::shared_ptr PathIdBytes; - const std::shared_ptr PathIdChunks; -public: - TPathIdClientCounters(std::shared_ptr pathIdCommittedBytes, std::shared_ptr pathIdCommittedChunks) - : PathIdBytes(pathIdCommittedBytes) - , PathIdChunks(pathIdCommittedChunks) { - } - - void OnPathIdDataInfo(const ui64 bytes, const ui32 chunks) const { - PathIdBytes->SetValue(bytes); - PathIdChunks->SetValue(chunks); - } -}; - -class TPathIdInfoCounters: public TCommonCountersOwner { -private: - using TBase = TCommonCountersOwner; - const std::shared_ptr PathIdBytes; - const std::shared_ptr PathIdChunks; -public: - TPathIdInfoCounters(const TString& countersName, const TString& dataName) - : TBase(countersName) - , PathIdBytes(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Bytes")) - , PathIdChunks(TBase::GetValueAutoAggregations("PathId/" + dataName + "/Chunks")) - { - } - - TPathIdClientCounters GetClient() const { - return TPathIdClientCounters(PathIdBytes->GetClient(), PathIdChunks->GetClient()); - } -}; - -class TPathIdOwnedCounters { -public: - const TPathIdClientCounters Inserted; - const TPathIdClientCounters Committed; - TPathIdOwnedCounters(TPathIdClientCounters&& inserted, TPathIdClientCounters&& committed) - : Inserted(std::move(inserted)) - , Committed(std::move(committed)) { - - } -}; - -class TInsertTableCounters: public TCommonCountersOwner { -private: - using TBase = TCommonCountersOwner; - const TPathIdInfoCounters PathIdInserted; - const TPathIdInfoCounters PathIdCommitted; -public: - const TDataOwnerSignals Inserted; - const TDataOwnerSignals Committed; - const TDataOwnerSignals Aborted; - const TLoadTimeSignals LoadCounters; - - TInsertTableCounters(); - - TPathIdOwnedCounters GetPathIdCounters() const { - return TPathIdOwnedCounters(PathIdInserted.GetClient(), PathIdCommitted.GetClient()); - } -}; - -} diff --git a/ydb/core/tx/columnshard/counters/tablet_counters.h b/ydb/core/tx/columnshard/counters/tablet_counters.h index 99292c4f73b8..8172278cf1bf 100644 --- a/ydb/core/tx/columnshard/counters/tablet_counters.h +++ b/ydb/core/tx/columnshard/counters/tablet_counters.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace NKikimr::NColumnShard { @@ -87,24 +86,12 @@ class TTabletCountersHandle { IncCounter(NColumnShard::COUNTER_READ_INDEX_BYTES, countersDelta.Bytes); } - void OnWriteCommitted(const NOlap::TInsertionSummary::TCounters& countersDelta) const { - IncCounter(COUNTER_BLOBS_COMMITTED, countersDelta.Rows); - IncCounter(COUNTER_BYTES_COMMITTED, countersDelta.Bytes); - IncCounter(COUNTER_RAW_BYTES_COMMITTED, countersDelta.RawBytes); - } - void OnCompactionWriteIndexCompleted(bool success, const ui64 blobsWritten, const ui64 bytesWritten) const { IncCounter(success ? NColumnShard::COUNTER_SPLIT_COMPACTION_SUCCESS : NColumnShard::COUNTER_SPLIT_COMPACTION_FAIL); IncCounter(NColumnShard::COUNTER_SPLIT_COMPACTION_BLOBS_WRITTEN, blobsWritten); IncCounter(NColumnShard::COUNTER_SPLIT_COMPACTION_BYTES_WRITTEN, bytesWritten); } - void OnInsertionWriteIndexCompleted(const ui64 blobsWritten, const ui64 bytesWritten, const TDuration duration) const { - IncCounter(NColumnShard::COUNTER_INDEXING_BLOBS_WRITTEN, blobsWritten); - IncCounter(NColumnShard::COUNTER_INDEXING_BYTES_WRITTEN, bytesWritten); - IncCounter(NColumnShard::COUNTER_INDEXING_TIME, duration.MilliSeconds()); - } - void OnWritePutBlobsSuccess(const ui64 rowsWritten) const { IncCounter(NColumnShard::COUNTER_OPERATIONS_ROWS_WRITTEN, rowsWritten); } diff --git a/ydb/core/tx/columnshard/counters/ya.make b/ydb/core/tx/columnshard/counters/ya.make index 5f54a69ba72b..1366b4cdb0e6 100644 --- a/ydb/core/tx/columnshard/counters/ya.make +++ b/ydb/core/tx/columnshard/counters/ya.make @@ -9,7 +9,6 @@ SRCS( common_data.cpp engine_logs.cpp indexation.cpp - insert_table.cpp req_tracer.cpp scan.cpp splitter.cpp diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_tables.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_tables.cpp index 34d7354b5124..c82d53ba7e24 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_tables.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_tables.cpp @@ -17,7 +17,6 @@ void TCleanupTablesColumnEngineChanges::DoDebugString(TStringOutput& out) const void TCleanupTablesColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { if (self && context.DB) { for (auto&& t : TablesToDrop) { - AFL_VERIFY(!self->InsertTable->HasDataInPathId(t)); AFL_VERIFY(self->TablesManager.TryFinalizeDropPathOnExecute(*context.DB, t)); } } @@ -25,7 +24,6 @@ void TCleanupTablesColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TCol void TCleanupTablesColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& /*context*/) { for (auto&& t : TablesToDrop) { - self->InsertTable->ErasePath(t); self->TablesManager.TryFinalizeDropPathOnComplete(t); } self->Subscribers->OnEvent(std::make_shared(TablesToDrop)); diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.cpp b/ydb/core/tx/columnshard/engines/changes/indexation.cpp deleted file mode 100644 index dc1f8bd00edc..000000000000 --- a/ydb/core/tx/columnshard/engines/changes/indexation.cpp +++ /dev/null @@ -1,300 +0,0 @@ -#include "indexation.h" - -#include "compaction/merger.h" - -#include - -namespace NKikimr::NOlap { - -void TInsertColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { - TBase::DoWriteIndexOnExecute(self, context); - if (self) { - auto removing = BlobsAction.GetRemoving(IStoragesManager::DefaultStorageId); - for (const auto& insertedData : DataToIndex) { - self->InsertTable->EraseCommittedOnExecute(context.DBWrapper, insertedData, removing); - } - } -} - -void TInsertColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { - TBase::DoStart(self); - Y_ABORT_UNLESS(DataToIndex.size()); - auto reading = BlobsAction.GetReading(IStoragesManager::DefaultStorageId); - for (auto&& insertedData : DataToIndex) { - reading->AddRange(insertedData.GetBlobRange(), insertedData.GetBlobData()); - } - - self.BackgroundController.StartIndexing(*this); -} - -void TInsertColumnEngineChanges::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { - TBase::DoWriteIndexOnComplete(self, context); - if (self) { - for (const auto& insertedData : DataToIndex) { - self->InsertTable->EraseCommittedOnComplete(insertedData); - } - if (!DataToIndex.empty()) { - self->UpdateInsertTableCounters(); - } - self->Counters.GetTabletCounters()->OnInsertionWriteIndexCompleted(context.BlobsWritten, context.BytesWritten, context.Duration); - } -} - -void TInsertColumnEngineChanges::DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& /*context*/) { - self.BackgroundController.FinishIndexing(*this); -} - -namespace { - -class TBatchInfo { -private: - YDB_READONLY_DEF(std::shared_ptr, Batch); - -public: - TBatchInfo(const std::shared_ptr& batch, const NEvWrite::EModificationType /*modificationType*/) - : Batch(batch) { - } -}; - -class TPathFieldsInfo { -private: - std::set UsageColumnIds; - const ISnapshotSchema::TPtr ResultSchema; - THashMap Schemas; - bool Finished = false; - const ui32 FullColumnsCount; - -public: - TPathFieldsInfo(const ISnapshotSchema::TPtr& resultSchema) - : UsageColumnIds(IIndexInfo::GetNecessarySystemColumnIdsSet()) - , ResultSchema(resultSchema) - , FullColumnsCount(ResultSchema->GetIndexInfo().GetColumnIds(true).size()) - { - AFL_VERIFY(FullColumnsCount); - } - - bool IsFinished() const { - return Finished; - } - - bool HasDeletion() const { - AFL_VERIFY(Finished); - return UsageColumnIds.contains((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG); - } - - void Finish() { - AFL_VERIFY(UsageColumnIds.size()); - AFL_VERIFY(!Finished); - Finished = true; - if (UsageColumnIds.size() == FullColumnsCount) { - return; - } - auto defaultDiffs = ISnapshotSchema::GetColumnsWithDifferentDefaults(Schemas, ResultSchema); - UsageColumnIds.insert(defaultDiffs.begin(), defaultDiffs.end()); - } - - const std::set& GetUsageColumnIds() const { - AFL_VERIFY(Finished); - return UsageColumnIds; - } - - void AddChunkInfo(const TCommittedData& data, const TConstructionContext& context) { - AFL_VERIFY(!Finished); - if (UsageColumnIds.size() == FullColumnsCount) { - return; - } - auto blobSchema = context.SchemaVersions.GetSchemaVerified(data.GetSchemaVersion()); - std::set columnIdsToDelete = blobSchema->GetColumnIdsToDelete(ResultSchema); - if (!Schemas.contains(data.GetSchemaVersion())) { - Schemas.emplace(data.GetSchemaVersion(), blobSchema); - } - TColumnIdsView columnIds = blobSchema->GetIndexInfo().GetColumnIds(false); - std::vector filteredIds = data.GetMeta().GetSchemaSubset().Apply(columnIds.begin(), columnIds.end()); - if (data.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete) { - filteredIds.emplace_back((ui32)IIndexInfo::ESpecialColumn::DELETE_FLAG); - } - for (const auto& filteredId : filteredIds) { - if (!columnIdsToDelete.contains(filteredId)) { - UsageColumnIds.insert(filteredId); - } - } - } -}; - -class TPathData { -private: - std::vector Batches; - YDB_READONLY_DEF(std::optional, ShardingInfo); - TPathFieldsInfo ColumnsInfo; - -public: - TPathData(const std::optional& shardingInfo, const ISnapshotSchema::TPtr& resultSchema) - : ShardingInfo(shardingInfo) - , ColumnsInfo(resultSchema) { - } - - const TPathFieldsInfo& GetColumnsInfo() const { - return ColumnsInfo; - } - - void FinishChunksInfo() { - ColumnsInfo.Finish(); - } - - std::vector> GetGeneralContainers() const { - std::vector> result; - for (auto&& i : Batches) { - result.emplace_back(i.GetBatch()); - } - return result; - } - - void AddChunkInfo(const NOlap::TCommittedData& data, const TConstructionContext& context) { - ColumnsInfo.AddChunkInfo(data, context); - } - - bool HasDeletion() { - return ColumnsInfo.HasDeletion(); - } - - void AddBatch(const NOlap::TCommittedData& data, const std::shared_ptr& batch) { - AFL_VERIFY(ColumnsInfo.IsFinished()); - AFL_VERIFY(batch); - Batches.emplace_back(batch, data.GetMeta().GetModificationType()); - } - - void AddShardingInfo(const std::optional& info) { - if (!info) { - ShardingInfo.reset(); - } else if (ShardingInfo && info->GetSnapshotVersion() < ShardingInfo->GetSnapshotVersion()) { - ShardingInfo = info; - } - } -}; - -class TPathesData { -private: - THashMap Data; - const ISnapshotSchema::TPtr ResultSchema; - -public: - TPathesData(const ISnapshotSchema::TPtr& resultSchema) - : ResultSchema(resultSchema) { - } - - void FinishChunksInfo() { - for (auto&& i : Data) { - i.second.FinishChunksInfo(); - } - } - - const THashMap& GetData() const { - return Data; - } - - void AddChunkInfo(const NOlap::TCommittedData& inserted, const TConstructionContext& context) { - auto shardingFilterCommit = context.SchemaVersions.GetShardingInfoOptional(inserted.GetPathId(), inserted.GetSnapshot()); - auto it = Data.find(inserted.GetPathId()); - if (it == Data.end()) { - it = Data.emplace(inserted.GetPathId(), TPathData(shardingFilterCommit, ResultSchema)).first; - } - it->second.AddChunkInfo(inserted, context); - it->second.AddShardingInfo(shardingFilterCommit); - } - - void AddBatch(const NOlap::TCommittedData& inserted, const std::shared_ptr& batch) { - auto it = Data.find(inserted.GetPathId()); - AFL_VERIFY(it != Data.end()); - it->second.AddBatch(inserted, batch); - } - - const TPathFieldsInfo& GetPathInfo(const TInternalPathId pathId) const { - auto it = Data.find(pathId); - AFL_VERIFY(it != Data.end()); - return it->second.GetColumnsInfo(); - } -}; - -} // namespace - -TConclusionStatus TInsertColumnEngineChanges::DoConstructBlobs(TConstructionContext& context) noexcept { - Y_ABORT_UNLESS(!DataToIndex.empty()); - Y_ABORT_UNLESS(AppendedPortions.empty()); - - auto resultSchema = context.SchemaVersions.GetLastSchema(); - Y_ABORT_UNLESS(resultSchema->GetIndexInfo().IsSorted()); - - TPathesData pathBatches(resultSchema); - for (auto& inserted : DataToIndex) { - if (inserted.GetRemove()) { - continue; - } - pathBatches.AddChunkInfo(inserted, context); - } - NoAppendIsCorrect = pathBatches.GetData().empty(); - - pathBatches.FinishChunksInfo(); - - for (auto& inserted : DataToIndex) { - const TBlobRange& blobRange = inserted.GetBlobRange(); - if (inserted.GetRemove()) { - Blobs.Extract(IStoragesManager::DefaultStorageId, blobRange); - continue; - } - auto blobSchema = context.SchemaVersions.GetSchemaVerified(inserted.GetSchemaVersion()); - - std::shared_ptr batch; - { - const auto blobData = Blobs.Extract(IStoragesManager::DefaultStorageId, blobRange); - - NArrow::TSchemaLiteView blobSchemaView = blobSchema->GetIndexInfo().ArrowSchema(); - auto batchSchema = - std::make_shared(inserted.GetMeta().GetSchemaSubset().Apply(blobSchemaView.begin(), blobSchemaView.end())); - batch = std::make_shared(NArrow::DeserializeBatch(blobData, batchSchema)); - std::set columnIdsToDelete = blobSchema->GetColumnIdsToDelete(resultSchema); - if (!columnIdsToDelete.empty()) { - batch->DeleteFieldsByIndex(blobSchema->ConvertColumnIdsToIndexes(columnIdsToDelete)); - } - } - IIndexInfo::AddSnapshotColumns(*batch, inserted.GetSnapshot(), (ui64)inserted.GetInsertWriteId()); - - auto& pathInfo = pathBatches.GetPathInfo(inserted.GetPathId()); - - if (pathInfo.HasDeletion()) { - IIndexInfo::AddDeleteFlagsColumn(*batch, inserted.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete); - } - - pathBatches.AddBatch(inserted, batch); - } - - Y_ABORT_UNLESS(Blobs.IsEmpty()); - auto stats = std::make_shared(); - std::vector> filters; - for (auto& [pathId, pathInfo] : pathBatches.GetData()) { - auto filteredSnapshot = std::make_shared(resultSchema, pathInfo.GetColumnsInfo().GetUsageColumnIds()); - std::optional shardingVersion; - if (pathInfo.GetShardingInfo()) { - shardingVersion = pathInfo.GetShardingInfo()->GetSnapshotVersion(); - } - auto batches = pathInfo.GetGeneralContainers(); - filters.resize(batches.size()); - - auto itGranule = PathToGranule.find(pathId); - AFL_VERIFY(itGranule != PathToGranule.end())("path_id", pathId); - NCompaction::TMerger merger(context, SaverContext, std::move(batches), std::move(filters)); - merger.SetOptimizationWritingPackMode(true); - auto localAppended = merger.Execute(stats, itGranule->second, filteredSnapshot, pathId, shardingVersion); - for (auto&& i : localAppended) { - AppendedPortions.emplace_back(std::move(i)); - } - } - - Y_ABORT_UNLESS(PathToGranule.size() == pathBatches.GetData().size()); - return TConclusionStatus::Success(); -} - -NColumnShard::ECumulativeCounters TInsertColumnEngineChanges::GetCounterIndex(const bool isSuccess) const { - return isSuccess ? NColumnShard::COUNTER_INDEXING_SUCCESS : NColumnShard::COUNTER_INDEXING_FAIL; -} - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/changes/indexation.h b/ydb/core/tx/columnshard/engines/changes/indexation.h deleted file mode 100644 index fcc4f845988c..000000000000 --- a/ydb/core/tx/columnshard/engines/changes/indexation.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once -#include "with_appended.h" - -#include "abstract/abstract.h" - -#include -#include -#include - -#include - -namespace NKikimr::NOlap { - -class TInsertColumnEngineChanges: public TChangesWithAppend, public NColumnShard::TMonitoringObjectsCounter { -private: - using TBase = TChangesWithAppend; - std::vector DataToIndex; - -protected: - virtual void DoWriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) override; - virtual void DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) override; - - virtual void DoStart(NColumnShard::TColumnShard& self) override; - virtual void DoOnFinish(NColumnShard::TColumnShard& self, TChangesFinishContext& context) override; - virtual TConclusionStatus DoConstructBlobs(TConstructionContext& context) noexcept override; - virtual NColumnShard::ECumulativeCounters GetCounterIndex(const bool isSuccess) const override; - virtual ui64 DoCalcMemoryForUsage() const override { - ui64 result = 0; - for (auto& ptr : DataToIndex) { - result += ptr.GetMeta().GetRawBytes(); - } - return result; - } - - virtual std::shared_ptr DoBuildDataLockImpl() const override { - return nullptr; - } - virtual NDataLocks::ELockCategory GetLockCategory() const override { - return NDataLocks::ELockCategory::Compaction; - } -public: - THashMap PathToGranule; // pathId -> positions (sorted by pk) -public: - TInsertColumnEngineChanges(std::vector&& dataToIndex, const TSaverContext& saverContext) - : TBase(saverContext, NBlobOperations::EConsumer::INDEXATION) - , DataToIndex(std::move(dataToIndex)) { - SetTargetCompactionLevel(0); - } - - const std::vector& GetDataToIndex() const { - return DataToIndex; - } - - static TString StaticTypeName() { - return "CS::INDEXATION"; - } - - virtual TString TypeString() const override { - return StaticTypeName(); - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/changes/ya.make b/ydb/core/tx/columnshard/engines/changes/ya.make index b0d9147ad80e..27e6c1554297 100644 --- a/ydb/core/tx/columnshard/engines/changes/ya.make +++ b/ydb/core/tx/columnshard/engines/changes/ya.make @@ -5,7 +5,6 @@ SRCS( cleanup_tables.cpp compaction.cpp general_compaction.cpp - indexation.cpp merge_subset.cpp ttl.cpp with_appended.cpp @@ -14,7 +13,6 @@ SRCS( PEERDIR( ydb/core/formats/arrow ydb/core/tx/columnshard/common - ydb/core/tx/columnshard/engines/insert_table ydb/core/tx/columnshard/engines/changes/abstract ydb/core/tx/columnshard/engines/changes/compaction ydb/core/tx/columnshard/engines/changes/counters diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index e6a268f49196..84bf975ec185 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -147,7 +147,6 @@ class IColumnEngine { virtual bool IsOverloadedByMetadata(const ui64 limit) const = 0; virtual std::shared_ptr Select( TInternalPathId pathId, TSnapshot snapshot, const TPKRangesFilter& pkRangesFilter, const bool withUncommitted) const = 0; - virtual std::shared_ptr StartInsert(std::vector&& dataToIndex) noexcept = 0; virtual std::shared_ptr StartCompaction(const std::shared_ptr& dataLocksManager) noexcept = 0; virtual ui64 GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, const std::optional waitingPriority) const noexcept = 0; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index f6bc38e08b55..c05783156fa2 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -6,7 +6,6 @@ #include "changes/cleanup_portions.h" #include "changes/cleanup_tables.h" #include "changes/general_compaction.h" -#include "changes/indexation.h" #include "changes/ttl.h" #include "loading/stages.h" @@ -169,27 +168,6 @@ bool TColumnEngineForLogs::FinishLoading() { return true; } -std::shared_ptr TColumnEngineForLogs::StartInsert(std::vector&& dataToIndex) noexcept { - Y_ABORT_UNLESS(dataToIndex.size()); - - TSaverContext saverContext(StoragesManager); - auto changes = std::make_shared(std::move(dataToIndex), saverContext); - auto pkSchema = VersionedIndex.GetLastSchema()->GetIndexInfo().GetReplaceKey(); - - for (const auto& data : changes->GetDataToIndex()) { - const TInternalPathId pathId = data.GetPathId(); - - if (changes->PathToGranule.contains(pathId)) { - continue; - } - if (!data.GetRemove()) { - AFL_VERIFY(changes->PathToGranule.emplace(pathId, GetGranulePtrVerified(pathId)->GetBucketPositions()).second); - } - } - - return changes; -} - ui64 TColumnEngineForLogs::GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, const std::optional waitingPriority) const noexcept { auto priority = GranulesStorage->GetCompactionPriority(dataLocksManager, pathIds, waitingPriority); diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 271d1a330158..38ffa33e60a3 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -145,7 +145,6 @@ class TColumnEngineForLogs: public IColumnEngine { virtual std::vector CollectMetadataRequests() const override { return GranulesStorage->CollectMetadataRequests(); } - std::shared_ptr StartInsert(std::vector&& dataToIndex) noexcept override; ui64 GetCompactionPriority(const std::shared_ptr& dataLocksManager, const std::set& pathIds, const std::optional waitingPriority) const noexcept override; std::shared_ptr StartCompaction(const std::shared_ptr& dataLocksManager) noexcept override; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 584fe3417cdf..008f17e41324 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -9,41 +9,6 @@ namespace NKikimr::NOlap { -void TDbWrapper::Insert(const TInsertedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_Insert(db, data); -} - -void TDbWrapper::Commit(const TCommittedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_Commit(db, data); -} - -void TDbWrapper::Abort(const TInsertedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_Abort(db, data); -} - -void TDbWrapper::EraseInserted(const TInsertedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_EraseInserted(db, data); -} - -void TDbWrapper::EraseCommitted(const TCommittedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_EraseCommitted(db, data); -} - -void TDbWrapper::EraseAborted(const TInsertedData& data) { - NIceDb::TNiceDb db(Database); - NColumnShard::Schema::InsertTable_EraseAborted(db, data); -} - -bool TDbWrapper::Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) { - NIceDb::TNiceDb db(Database); - return NColumnShard::Schema::InsertTable_Load(db, DsGroupSelector, insertTable, loadTime); -} - void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) { if (!AppDataVerified().ColumnShardConfig.GetColumnChunksV1Usage() && !AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { return; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index a0040cb40756..d1848317305f 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -17,9 +17,6 @@ namespace NKikimr::NOlap { class TColumnChunkLoadContextV2; class TIndexChunkLoadContext; -class TInsertedData; -class TCommittedData; -class TInsertTableAccessor; class TColumnRecord; class TIndexChunk; struct TGranuleRecord; @@ -38,16 +35,8 @@ class IDbWrapper { return *result; } - virtual void Insert(const TInsertedData& data) = 0; - virtual void Commit(const TCommittedData& data) = 0; - virtual void Abort(const TInsertedData& data) = 0; - virtual void EraseInserted(const TInsertedData& data) = 0; - virtual void EraseCommitted(const TCommittedData& data) = 0; - virtual void EraseAborted(const TInsertedData& data) = 0; virtual void WriteColumns(const NOlap::TPortionInfo& portion, const NKikimrTxColumnShard::TIndexPortionAccessor& proto) = 0; - virtual bool Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) = 0; - virtual void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0; virtual void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) = 0; virtual bool LoadColumns(const std::optional pathId, const std::function& callback) = 0; @@ -75,15 +64,6 @@ class TDbWrapper : public IDbWrapper { , DsGroupSelector(dsGroupSelector) {} - void Insert(const TInsertedData& data) override; - void Commit(const TCommittedData& data) override; - void Abort(const TInsertedData& data) override; - void EraseInserted(const TInsertedData& data) override; - void EraseCommitted(const TCommittedData& data) override; - void EraseAborted(const TInsertedData& data) override; - - bool Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) override; - void WritePortion(const NOlap::TPortionInfo& portion) override; void ErasePortion(const NOlap::TPortionInfo& portion) override; bool LoadPortions(const std::optional pathId, const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& callback) override; diff --git a/ydb/core/tx/columnshard/engines/insert_table/committed.cpp b/ydb/core/tx/columnshard/engines/insert_table/committed.cpp deleted file mode 100644 index bd4bb9ff6c06..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/committed.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "committed.h" - -namespace NKikimr::NOlap { - -} diff --git a/ydb/core/tx/columnshard/engines/insert_table/committed.h b/ydb/core/tx/columnshard/engines/insert_table/committed.h deleted file mode 100644 index f1cb82e7e5ae..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/committed.h +++ /dev/null @@ -1,163 +0,0 @@ -#pragma once -#include "user_data.h" - -#include - -namespace NKikimr::NOlap { - -class TCommittedData: public TUserDataContainer { -private: - using TBase = TUserDataContainer; - YDB_READONLY(TSnapshot, Snapshot, NOlap::TSnapshot::Zero()); - YDB_READONLY(TInsertWriteId, InsertWriteId, (TInsertWriteId)0); - YDB_READONLY_DEF(TString, DedupId); - YDB_READONLY(bool, Remove, false); - -public: - TCommittedData(const std::shared_ptr& userData, const ui64 planStep, const ui64 txId, const TInsertWriteId insertWriteId) - : TBase(userData) - , Snapshot(planStep, txId) - , InsertWriteId(insertWriteId) - , DedupId(ToString(planStep) + ":" + ToString((ui64)insertWriteId)) { - } - - TCommittedData(const std::shared_ptr& userData, const ui64 planStep, const ui64 txId, const TInsertWriteId insertWriteId, - const TString& dedupId) - : TBase(userData) - , Snapshot(planStep, txId) - , InsertWriteId(insertWriteId) - , DedupId(dedupId) { - } - - TCommittedData(const std::shared_ptr& userData, const TSnapshot& ss, const ui64 generation, const TInsertWriteId ephemeralWriteId) - : TBase(userData) - , Snapshot(ss) - , InsertWriteId(ephemeralWriteId) - , DedupId(ToString(generation) + ":" + ToString((ui64)ephemeralWriteId)) { - } - - void SetRemove() { - AFL_VERIFY(!Remove); - Remove = true; - } - - bool operator<(const TCommittedData& key) const { - if (Snapshot == key.Snapshot) { - if (UserData->GetPathId() == key.UserData->GetPathId()) { - return DedupId < key.DedupId; - } else { - return UserData->GetPathId() < key.UserData->GetPathId(); - } - } else { - return Snapshot < key.Snapshot; - } - } -}; - -class TCommittedBlob { -private: - TBlobRange BlobRange; - std::optional CommittedSnapshot; - const TInsertWriteId InsertWriteId; - YDB_READONLY(ui64, SchemaVersion, 0); - YDB_READONLY(ui64, RecordsCount, 0); - YDB_READONLY(bool, IsDelete, false); - NArrow::TSimpleRow First; - NArrow::TSimpleRow Last; - YDB_READONLY_DEF(NArrow::TSchemaSubset, SchemaSubset); - -public: - const std::optional& GetCommittedSnapshot() const { - return CommittedSnapshot; - } - - const TSnapshot& GetCommittedSnapshotDef(const TSnapshot& def) const { - if (CommittedSnapshot) { - return *CommittedSnapshot; - } else { - return def; - } - } - - const TSnapshot& GetCommittedSnapshotVerified() const { - AFL_VERIFY(!!CommittedSnapshot); - return *CommittedSnapshot; - } - - bool IsCommitted() const { - return !!CommittedSnapshot; - } - - TInsertWriteId GetInsertWriteId() const { - return InsertWriteId; - } - - const NArrow::TSimpleRow& GetFirst() const { - return First; - } - const NArrow::TSimpleRow& GetLast() const { - return Last; - } - - ui64 GetSize() const { - return BlobRange.Size; - } - - TCommittedBlob(const TBlobRange& blobRange, const TSnapshot& snapshot, const TInsertWriteId insertWriteId, const ui64 schemaVersion, - const ui64 recordsCount, const NArrow::TSimpleRow& first, const NArrow::TSimpleRow& last, const bool isDelete, - const NArrow::TSchemaSubset& subset) - : BlobRange(blobRange) - , CommittedSnapshot(snapshot) - , InsertWriteId(insertWriteId) - , SchemaVersion(schemaVersion) - , RecordsCount(recordsCount) - , IsDelete(isDelete) - , First(first) - , Last(last) - , SchemaSubset(subset) { - } - - TCommittedBlob(const TBlobRange& blobRange, const TInsertWriteId insertWriteId, const ui64 schemaVersion, const ui64 recordsCount, - const NArrow::TSimpleRow& first, const NArrow::TSimpleRow& last, const bool isDelete, - const NArrow::TSchemaSubset& subset) - : BlobRange(blobRange) - , InsertWriteId(insertWriteId) - , SchemaVersion(schemaVersion) - , RecordsCount(recordsCount) - , IsDelete(isDelete) - , First(first) - , Last(last) - , SchemaSubset(subset) { - } - - /// It uses trick then we place key with planStep:txId in container and find them later by BlobId only. - /// So hash() and equality should depend on BlobId only. - bool operator==(const TCommittedBlob& key) const { - return BlobRange == key.BlobRange; - } - ui64 Hash() const noexcept { - return BlobRange.Hash(); - } - TString DebugString() const { - TStringBuilder sb; - sb << BlobRange; - if (CommittedSnapshot) { - sb << ";snapshot=" << CommittedSnapshot->DebugString(); - } - sb << ";write_id=" << GetInsertWriteId(); - return sb; - } - - const TBlobRange& GetBlobRange() const { - return BlobRange; - } -}; - -} // namespace NKikimr::NOlap - -template <> -struct THash { - inline size_t operator()(const NKikimr::NOlap::TCommittedBlob& key) const { - return key.Hash(); - } -}; diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp deleted file mode 100644 index 8be5b6344c94..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ /dev/null @@ -1,218 +0,0 @@ -#include "insert_table.h" - -#include -#include -#include -#include - -namespace NKikimr::NOlap { - -bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { - if (auto* dataPtr = Summary.AddInserted(std::move(data))) { - AddBlobLink(dataPtr->GetBlobRange().BlobId); - dbTable.Insert(*dataPtr); - return true; - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_insertion"); - return false; - } -} - -TInsertionSummary::TCounters TInsertTable::Commit( - IDbWrapper& dbTable, ui64 planStep, ui64 txId, const THashSet& writeIds, std::function pathExists) { - Y_ABORT_UNLESS(!writeIds.empty()); - - TInsertionSummary::TCounters counters; - for (auto writeId : writeIds) { - std::optional data = Summary.ExtractInserted(writeId); - if (!data) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("ps", planStep)("tx", txId)("write_id", (ui64)writeId)("event", "hasn't data for commit"); - continue; - } - - counters.Rows += data->GetMeta().GetRecordsCount(); - counters.RawBytes += data->GetMeta().GetRawBytes(); - counters.Bytes += data->BlobSize(); - - dbTable.EraseInserted(*data); - - const TInternalPathId pathId = data->GetPathId(); - auto* pathInfo = Summary.GetPathInfoOptional(pathId); - // There could be commit after drop: propose, drop, plan - if (pathInfo && pathExists(pathId)) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "commit_insertion")("path_id", data->GetPathId())( - "blob_range", data->GetBlobRange().ToString()); - auto committed = data->Commit(planStep, txId); - dbTable.Commit(committed); - - pathInfo->AddCommitted(std::move(committed)); - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( - "blob_range", data->GetBlobRange().ToString()); - dbTable.Abort(*data); - Summary.AddAborted(std::move(*data)); - } - } - - return counters; -} - -TInsertionSummary::TCounters TInsertTable::CommitEphemeral(IDbWrapper& dbTable, TCommittedData&& data) { - TInsertionSummary::TCounters counters; - counters.Rows += data.GetMeta().GetRecordsCount(); - counters.RawBytes += data.GetMeta().GetRawBytes(); - counters.Bytes += data.BlobSize(); - - AddBlobLink(data.GetBlobRange().BlobId); - const TInternalPathId pathId = data.GetPathId(); - auto& pathInfo = Summary.GetPathInfoVerified(pathId); - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "commit_insertion")("path_id", pathId)("blob_range", data.GetBlobRange().ToString()); - dbTable.Commit(data); - pathInfo.AddCommitted(std::move(data)); - - return counters; -} - -void TInsertTable::Abort(IDbWrapper& dbTable, const THashSet& writeIds) { - Y_ABORT_UNLESS(!writeIds.empty()); - - for (auto writeId : writeIds) { - // There could be inconsistency with txs and writes in case of bugs. So we could find no record for writeId. - if (std::optional data = Summary.ExtractInserted(writeId)) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( - "blob_range", data->GetBlobRange().ToString())("write_id", writeId); - dbTable.EraseInserted(*data); - dbTable.Abort(*data); - Summary.AddAborted(std::move(*data)); - } - } -} - -THashSet TInsertTable::OldWritesToAbort(const TInstant& now) const { - return Summary.GetExpiredInsertions(now - WaitCommitDelay, CleanupPackageSize); -} - -void TInsertTable::EraseCommittedOnExecute( - IDbWrapper& dbTable, const TCommittedData& data, const std::shared_ptr& blobsAction) { - if (Summary.HasCommitted(data)) { - dbTable.EraseCommitted(data); - RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); - } -} - -void TInsertTable::EraseCommittedOnComplete(const TCommittedData& data) { - if (Summary.EraseCommitted(data)) { - RemoveBlobLinkOnComplete(data.GetBlobRange().BlobId); - } -} - -void TInsertTable::EraseAbortedOnExecute( - IDbWrapper& dbTable, const TInsertedData& data, const std::shared_ptr& blobsAction) { - if (Summary.HasAborted(data.GetInsertWriteId())) { - dbTable.EraseAborted(data); - RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); - } -} - -void TInsertTable::EraseAbortedOnComplete(const TInsertedData& data) { - if (Summary.EraseAborted(data.GetInsertWriteId())) { - RemoveBlobLinkOnComplete(data.GetBlobRange().BlobId); - } -} - -bool TInsertTable::Load(NIceDb::TNiceDb& db, IDbWrapper& dbTable, const TInstant loadTime) { - Y_ABORT_UNLESS(!Loaded); - Loaded = true; - LastWriteId = (TInsertWriteId)0; - { - NColumnShard::TLoadTimeSignals::TLoadTimer timer = Summary.GetCounters().LoadCounters.StartGuard(); - if (!NColumnShard::Schema::GetSpecialValueOpt(db, NColumnShard::Schema::EValueIds::LastWriteId, LastWriteId)) { - timer.AddLoadingFail(); - return false; - } - - if (!dbTable.Load(*this, loadTime)) { - timer.AddLoadingFail(); - return false; - } - return true; - } -} - -std::vector TInsertTable::Read(TInternalPathId pathId, const std::optional lockId, const TSnapshot& reqSnapshot, - const std::shared_ptr& pkSchema, const TPKRangesFilter* pkRangesFilter) const { - const TPathInfo* pInfo = Summary.GetPathInfoOptional(pathId); - if (!pInfo) { - return {}; - } - - std::vector result; - result.reserve(pInfo->GetCommitted().size() + Summary.GetInserted().size()); - - for (const auto& data : pInfo->GetCommitted()) { - if (lockId || data.GetSnapshot() <= reqSnapshot) { - auto start = data.GetMeta().GetFirstPK(pkSchema); - auto finish = data.GetMeta().GetLastPK(pkSchema); - if (pkRangesFilter && !pkRangesFilter->IsUsed(start, finish)) { - continue; - } - result.emplace_back(TCommittedBlob(data.GetBlobRange(), data.GetSnapshot(), data.GetInsertWriteId(), data.GetSchemaVersion(), data.GetMeta().GetRecordsCount(), - start, finish, data.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete, data.GetMeta().GetSchemaSubset())); - } - } - if (lockId) { - for (const auto& [writeId, data] : Summary.GetInserted()) { - if (data.GetPathId() != pathId) { - continue; - } - auto start = data.GetMeta().GetFirstPK(pkSchema); - auto finish = data.GetMeta().GetLastPK(pkSchema); - if (pkRangesFilter && !pkRangesFilter->IsUsed(start, finish)) { - continue; - } - result.emplace_back(TCommittedBlob(data.GetBlobRange(), writeId, data.GetSchemaVersion(), data.GetMeta().GetRecordsCount(), start, finish, - data.GetMeta().GetModificationType() == NEvWrite::EModificationType::Delete, data.GetMeta().GetSchemaSubset())); - } - } - return result; -} - -TInsertWriteId TInsertTable::BuildNextWriteId(NTabletFlatExecutor::TTransactionContext& txc) { - NIceDb::TNiceDb db(txc.DB); - return BuildNextWriteId(db); -} - -TInsertWriteId TInsertTable::BuildNextWriteId(NIceDb::TNiceDb& db) { - TInsertWriteId writeId = ++LastWriteId; - NColumnShard::Schema::SaveSpecialValue(db, NColumnShard::Schema::EValueIds::LastWriteId, (ui64)writeId); - return writeId; -} - -bool TInsertTableAccessor::RemoveBlobLinkOnExecute( - const TUnifiedBlobId& blobId, const std::shared_ptr& blobsAction) { - AFL_VERIFY(blobsAction); - auto itBlob = BlobLinks.find(blobId); - AFL_VERIFY(itBlob != BlobLinks.end()); - AFL_VERIFY(itBlob->second >= 1); - if (itBlob->second == 1) { - blobsAction->DeclareSelfRemove(itBlob->first); - return true; - } else { - return false; - } -} - -bool TInsertTableAccessor::RemoveBlobLinkOnComplete(const TUnifiedBlobId& blobId) { - auto itBlob = BlobLinks.find(blobId); - AFL_VERIFY(itBlob != BlobLinks.end()); - AFL_VERIFY(itBlob->second >= 1); - if (itBlob->second == 1) { - BlobLinks.erase(itBlob); - return true; - } else { - --itBlob->second; - return false; - } -} - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h b/ydb/core/tx/columnshard/engines/insert_table/insert_table.h deleted file mode 100644 index 15f317e7776a..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h +++ /dev/null @@ -1,129 +0,0 @@ -#pragma once -#include "committed.h" -#include "inserted.h" -#include "path_info.h" -#include "rt_insertion.h" - -#include -#include -#include -#include -#include - -namespace NKikimr::NOlap { -class TPKRangesFilter; -class IDbWrapper; - -class TInsertTableAccessor { -protected: - TInsertionSummary Summary; - THashMap BlobLinks; - - void AddBlobLink(const TUnifiedBlobId& blobId) { - ++BlobLinks[blobId]; - } - - bool RemoveBlobLinkOnExecute(const TUnifiedBlobId& blobId, const std::shared_ptr& blobsAction); - bool RemoveBlobLinkOnComplete(const TUnifiedBlobId& blobId); - -public: - TPathInfo& RegisterPathInfo(const TInternalPathId pathId) { - return Summary.RegisterPathInfo(pathId); - } - - void ErasePath(const TInternalPathId pathId) { - Summary.ErasePath(pathId); - } - bool HasDataInPathId(const TInternalPathId pathId) const { - return Summary.HasPathIdData(pathId); - } - const std::map>& GetPathPriorities() const { - return Summary.GetPathPriorities(); - } - - std::optional GetMinCommittedSnapshot(const TInternalPathId pathId) const { - auto* info = Summary.GetPathInfoOptional(pathId); - if (!info) { - return {}; - } else if (info->GetCommitted().empty()) { - return {}; - } else { - return info->GetCommitted().begin()->GetSnapshot(); - } - } - - bool AddInserted(TInsertedData&& data, const bool load) { - if (load) { - AddBlobLink(data.GetBlobRange().BlobId); - } - return Summary.AddInserted(std::move(data), load); - } - bool AddAborted(TInsertedData&& data, const bool load) { - AFL_VERIFY_DEBUG(!Summary.ExtractInserted(data.GetInsertWriteId())); - if (load) { - AddBlobLink(data.GetBlobRange().BlobId); - } - return Summary.AddAborted(std::move(data), load); - } - bool AddCommitted(TCommittedData&& data, const bool load) { - if (load) { - AddBlobLink(data.GetBlobRange().BlobId); - } - const TInternalPathId pathId = data.GetPathId(); - return Summary.GetPathInfoVerified(pathId).AddCommitted(std::move(data), load); - } - bool HasPathIdData(const TInternalPathId pathId) const { - return Summary.HasPathIdData(pathId); - } - const THashMap& GetAborted() const { - return Summary.GetAborted(); - } - const TInsertedContainer& GetInserted() const { - return Summary.GetInserted(); - } - const TInsertionSummary::TCounters& GetCountersPrepared() const { - return Summary.GetCountersPrepared(); - } - const TInsertionSummary::TCounters& GetCountersCommitted() const { - return Summary.GetCountersCommitted(); - } - bool IsOverloadedByCommitted(const TInternalPathId pathId) const { - return Summary.IsOverloaded(pathId); - } -}; - -class TInsertTable: public TInsertTableAccessor { -private: - bool Loaded = false; - TInsertWriteId LastWriteId = TInsertWriteId{ 0 }; - -public: - static constexpr const TDuration WaitCommitDelay = TDuration::Minutes(10); - static constexpr ui64 CleanupPackageSize = 10000; - - bool Insert(IDbWrapper& dbTable, TInsertedData&& data); - TInsertionSummary::TCounters Commit( - IDbWrapper& dbTable, ui64 planStep, ui64 txId, const THashSet& writeIds, std::function pathExists); - TInsertionSummary::TCounters CommitEphemeral(IDbWrapper& dbTable, TCommittedData&& data); - void Abort(IDbWrapper& dbTable, const THashSet& writeIds); - void MarkAsNotAbortable(const TInsertWriteId writeId) { - Summary.MarkAsNotAbortable(writeId); - } - THashSet OldWritesToAbort(const TInstant& now) const; - - void EraseCommittedOnExecute( - IDbWrapper& dbTable, const TCommittedData& key, const std::shared_ptr& blobsAction); - void EraseCommittedOnComplete(const TCommittedData& key); - - void EraseAbortedOnExecute(IDbWrapper& dbTable, const TInsertedData& key, const std::shared_ptr& blobsAction); - void EraseAbortedOnComplete(const TInsertedData& key); - - std::vector Read(TInternalPathId pathId, const std::optional lockId, const TSnapshot& reqSnapshot, - const std::shared_ptr& pkSchema, const TPKRangesFilter* pkRangesFilter) const; - bool Load(NIceDb::TNiceDb& db, IDbWrapper& dbTable, const TInstant loadTime); - - TInsertWriteId BuildNextWriteId(NTabletFlatExecutor::TTransactionContext& txc); - TInsertWriteId BuildNextWriteId(NIceDb::TNiceDb& db); -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/inserted.cpp b/ydb/core/tx/columnshard/engines/insert_table/inserted.cpp deleted file mode 100644 index abf866eaae82..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/inserted.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "committed.h" -#include "inserted.h" - -#include - -namespace NKikimr::NOlap { - -TCommittedData TInsertedData::Commit(const ui64 planStep, const ui64 txId) const { - return TCommittedData(UserData, planStep, txId, InsertWriteId); -} - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/inserted.h b/ydb/core/tx/columnshard/engines/insert_table/inserted.h deleted file mode 100644 index 74750297b87c..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/inserted.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once -#include "user_data.h" - -#include - -namespace NKikimr::NOlap { - -class TCommittedData; - -class TInsertedData: public TUserDataContainer { -private: - using TBase = TUserDataContainer; - YDB_READONLY(TInsertWriteId, InsertWriteId, TInsertWriteId(0)); - YDB_READONLY_FLAG(NotAbortable, false); - -public: - void MarkAsNotAbortable() { - NotAbortableFlag = true; - } - - TInsertedData() = delete; // avoid invalid TInsertedData anywhere - - TInsertedData(const TInsertWriteId writeId, const std::shared_ptr& userData) - : TBase(userData) - , InsertWriteId(writeId) { - } - - /// We commit many writeIds in one txId. There could be several blobs with same WriteId and different DedupId. - /// One of them wins and becomes committed. Original DedupId would be lost then. - /// After commit we use original Initiator:WriteId as DedupId of inserted blob inside {PlanStep, TxId}. - /// pathId, initiator, {writeId}, {dedupId} -> pathId, planStep, txId, {dedupId} - [[nodiscard]] TCommittedData Commit(const ui64 planStep, const ui64 txId) const; -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/meta.cpp b/ydb/core/tx/columnshard/engines/insert_table/meta.cpp deleted file mode 100644 index eee7b2444b2e..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/meta.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "meta.h" - -namespace NKikimr::NOlap { - -NKikimrTxColumnShard::TLogicalMetadata TInsertedDataMeta::SerializeToProto() const { - return OriginalProto; -} - -std::shared_ptr TInsertedDataMeta::GetSpecialKeys(const std::shared_ptr& schema) const { - if (KeyInitialized.Val()) { - return SpecialKeysParsed; - } - std::shared_ptr result; - if (OriginalProto.HasSpecialKeysPayloadData()) { - result = std::make_shared(OriginalProto.GetSpecialKeysPayloadData(), schema); - } else if (OriginalProto.HasSpecialKeysRawData()) { - result = std::make_shared(OriginalProto.GetSpecialKeysRawData()); - } else { - AFL_VERIFY(false); - } - if (AtomicCas(&KeyInitialization, 1, 0)) { - SpecialKeysParsed = result; - KeyInitialized = 1; - } - return result; -} - -} diff --git a/ydb/core/tx/columnshard/engines/insert_table/meta.h b/ydb/core/tx/columnshard/engines/insert_table/meta.h deleted file mode 100644 index ea225efe143d..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/meta.h +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include - -#include -#include - -namespace NKikimr::NOlap { - -class TInsertedDataMeta { -private: - YDB_READONLY_DEF(TInstant, DirtyWriteTime); - YDB_READONLY(ui32, RecordsCount, 0); - YDB_READONLY(ui64, RawBytes, 0); - YDB_READONLY(NEvWrite::EModificationType, ModificationType, NEvWrite::EModificationType::Upsert); - YDB_READONLY_DEF(NArrow::TSchemaSubset, SchemaSubset); - - mutable TAtomicCounter KeyInitialized = 0; - mutable TAtomic KeyInitialization = 0; - mutable std::shared_ptr SpecialKeysParsed; - NKikimrTxColumnShard::TLogicalMetadata OriginalProto; - std::shared_ptr GetSpecialKeys(const std::shared_ptr& schema) const; - -public: - ui64 GetTxVolume() const { - return 512 + 2 * sizeof(ui64) + sizeof(ui32) + sizeof(OriginalProto) + (SpecialKeysParsed ? SpecialKeysParsed->GetMemorySize() : 0) + - SchemaSubset.GetTxVolume(); - } - - TInsertedDataMeta(const NKikimrTxColumnShard::TLogicalMetadata& proto) - : OriginalProto(proto) { - AFL_VERIFY(proto.HasDirtyWriteTimeSeconds())("data", proto.DebugString()); - DirtyWriteTime = TInstant::Seconds(proto.GetDirtyWriteTimeSeconds()); - RecordsCount = proto.GetNumRows(); - RawBytes = proto.GetRawBytes(); - if (proto.HasModificationType()) { - ModificationType = TEnumOperator::DeserializeFromProto(proto.GetModificationType()); - } - if (proto.HasSchemaSubset()) { - SchemaSubset.DeserializeFromProto(proto.GetSchemaSubset()).Validate(); - } - } - - NArrow::TSimpleRow GetFirstPK(const std::shared_ptr& schema) const { - AFL_VERIFY(schema); - return GetSpecialKeys(schema)->GetFirst(); - } - NArrow::TSimpleRow GetLastPK(const std::shared_ptr& schema) const { - AFL_VERIFY(schema); - return GetSpecialKeys(schema)->GetLast(); - } - - NKikimrTxColumnShard::TLogicalMetadata SerializeToProto() const; -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp b/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp deleted file mode 100644 index 304bc4ca4043..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/path_info.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "path_info.h" -#include "rt_insertion.h" -#include - -namespace NKikimr::NOlap { - -bool TPathInfo::SetCommittedOverload(const bool value) { - const bool startOverloaded = IsOverloaded(); - CommittedOverload = value; - return startOverloaded != IsOverloaded(); -} - -bool TPathInfo::SetInsertedOverload(const bool value) { - const bool startOverloaded = IsOverloaded(); - InsertedOverload = value; - return startOverloaded != IsOverloaded(); -} - -void TPathInfo::AddCommittedSize(const i64 size, const ui64 overloadLimit) { - CommittedSize += size; - Y_ABORT_UNLESS(CommittedSize >= 0); - SetCommittedOverload((ui64)CommittedSize > overloadLimit); -} - -void TPathInfo::AddInsertedSize(const i64 size, const ui64 overloadLimit) { - InsertedSize += size; - Y_ABORT_UNLESS(InsertedSize >= 0); - PathIdCounters.Inserted.OnPathIdDataInfo(InsertedSize, 0); - SetInsertedOverload((ui64)InsertedSize > overloadLimit); -} - -bool TPathInfo::EraseCommitted(const TCommittedData& data) { - Summary->RemovePriority(*this); - const bool result = Committed.erase(data); - AddCommittedSize(-1 * (i64)data.BlobSize(), TCompactionLimits::OVERLOAD_INSERT_TABLE_SIZE_BY_PATH_ID); - Summary->AddPriority(*this); - PathIdCounters.Committed.OnPathIdDataInfo(CommittedSize, Committed.size()); - Summary->OnEraseCommitted(*this, data.BlobSize()); - return result; -} - -bool TPathInfo::HasCommitted(const TCommittedData& data) { - return Committed.contains(data); -} - -bool TPathInfo::AddCommitted(TCommittedData&& data, const bool load) { - const ui64 dataSize = data.BlobSize(); - Summary->RemovePriority(*this); - AddCommittedSize(data.BlobSize(), TCompactionLimits::OVERLOAD_INSERT_TABLE_SIZE_BY_PATH_ID); - bool result = Committed.emplace(std::move(data)).second; - Summary->AddPriority(*this); - Summary->OnNewCommitted(dataSize, load); - PathIdCounters.Committed.OnPathIdDataInfo(CommittedSize, Committed.size()); - return result; -} - -TPathInfo::TPathInfo(TInsertionSummary& summary, const TInternalPathId pathId) - : PathId(pathId) - , Summary(&summary) - , PathIdCounters(Summary->GetCounters().GetPathIdCounters()) -{ - -} - -NKikimr::NOlap::TPathInfoIndexPriority TPathInfo::GetIndexationPriority() const { - if (CommittedSize > (i64)TCompactionLimits::WARNING_INSERT_TABLE_SIZE_BY_PATH_ID) { - return TPathInfoIndexPriority(TPathInfoIndexPriority::EIndexationPriority::PreventOverload, CommittedSize); - } else if (Committed.size() > TCompactionLimits::WARNING_INSERT_TABLE_COUNT_BY_PATH_ID) { - return TPathInfoIndexPriority(TPathInfoIndexPriority::EIndexationPriority::PreventManyPortions, Committed.size()); - } else { - return TPathInfoIndexPriority(TPathInfoIndexPriority::EIndexationPriority::NoPriority, CommittedSize * Committed.size() * Committed.size()); - } -} - -} diff --git a/ydb/core/tx/columnshard/engines/insert_table/path_info.h b/ydb/core/tx/columnshard/engines/insert_table/path_info.h deleted file mode 100644 index cb0c96a1ea01..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/path_info.h +++ /dev/null @@ -1,86 +0,0 @@ -#pragma once -#include "committed.h" -#include "inserted.h" - -#include -#include - -#include -#include - -namespace NKikimr::NOlap { -class TInsertionSummary; - -class TPathInfoIndexPriority { -public: - enum class EIndexationPriority { - PreventOverload = 100, - PreventManyPortions = 50, - NoPriority = 0 - }; - -private: - YDB_READONLY(EIndexationPriority, Category, EIndexationPriority::NoPriority); - const ui32 Weight; - -public: - TPathInfoIndexPriority(const EIndexationPriority category, const ui32 weight) - : Category(category) - , Weight(weight) { - } - - bool operator!() const { - return !Weight; - } - - bool operator<(const TPathInfoIndexPriority& item) const { - return std::tie(Category, Weight) < std::tie(item.Category, item.Weight); - } -}; - -class TPathInfo: public TMoveOnly { -private: - const TInternalPathId PathId; - TSet Committed; - YDB_READONLY(i64, CommittedSize, 0); - YDB_READONLY(i64, InsertedSize, 0); - bool CommittedOverload = false; - bool InsertedOverload = false; - TInsertionSummary* Summary = nullptr; - const NColumnShard::TPathIdOwnedCounters PathIdCounters; - - bool SetCommittedOverload(const bool value); - bool SetInsertedOverload(const bool value); - - void AddCommittedSize(const i64 size, const ui64 overloadLimit); - -public: - bool IsEmpty() const { - return Committed.empty() && !InsertedSize; - } - - void AddInsertedSize(const i64 size, const ui64 overloadLimit); - - explicit TPathInfo(TInsertionSummary& summary, const TInternalPathId pathId); - - TInternalPathId GetPathId() const { - return PathId; - } - - TPathInfoIndexPriority GetIndexationPriority() const; - - bool EraseCommitted(const TCommittedData& data); - bool HasCommitted(const TCommittedData& data); - - const TSet& GetCommitted() const { - return Committed; - } - - bool AddCommitted(TCommittedData&& data, const bool load = false); - - bool IsOverloaded() const { - return CommittedOverload || InsertedOverload; - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.cpp b/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.cpp deleted file mode 100644 index 87f803998088..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.cpp +++ /dev/null @@ -1,164 +0,0 @@ -#include "rt_insertion.h" -#include -#include - -namespace NKikimr::NOlap { - -void TInsertionSummary::OnNewCommitted(const ui64 dataSize, const bool load) noexcept { - Counters.Committed.Add(dataSize, load); - ++StatsCommitted.Rows; - StatsCommitted.Bytes += dataSize; - Y_ABORT_UNLESS(Counters.Committed.GetDataSize() == (i64)StatsCommitted.Bytes); -} - -void TInsertionSummary::OnEraseCommitted(TPathInfo& /*pathInfo*/, const ui64 dataSize) noexcept { - Counters.Committed.Erase(dataSize); - Y_ABORT_UNLESS(--StatsCommitted.Rows >= 0); - Y_ABORT_UNLESS(StatsCommitted.Bytes >= dataSize); - StatsCommitted.Bytes -= dataSize; - Y_ABORT_UNLESS(Counters.Committed.GetDataSize() == (i64)StatsCommitted.Bytes); -} - -void TInsertionSummary::RemovePriority(const TPathInfo& pathInfo) noexcept { - const auto priority = pathInfo.GetIndexationPriority(); - auto it = Priorities.find(priority); - if (it == Priorities.end()) { - AFL_VERIFY(!priority); - return; - } - AFL_VERIFY(!!priority); - Y_ABORT_UNLESS(it->second.erase(&pathInfo) || !priority); - if (it->second.empty()) { - Priorities.erase(it); - } -} - -void TInsertionSummary::AddPriority(const TPathInfo& pathInfo) noexcept { - if (!!pathInfo.GetIndexationPriority()) { - Y_ABORT_UNLESS(Priorities[pathInfo.GetIndexationPriority()].emplace(&pathInfo).second); - } -} - -TPathInfo& TInsertionSummary::RegisterPathInfo(const TInternalPathId pathId) { - auto it = PathInfo.find(pathId); - if (it == PathInfo.end()) { - it = PathInfo.emplace(pathId, TPathInfo(*this, pathId)).first; - } - return it->second; -} - -TPathInfo* TInsertionSummary::GetPathInfoOptional(const TInternalPathId pathId) { - auto it = PathInfo.find(pathId); - if (it == PathInfo.end()) { - return nullptr; - } - return &it->second; -} - -const TPathInfo* TInsertionSummary::GetPathInfoOptional(const TInternalPathId pathId) const { - auto it = PathInfo.find(pathId); - if (it == PathInfo.end()) { - return nullptr; - } - return &it->second; -} - -bool TInsertionSummary::IsOverloaded(const TInternalPathId pathId) const { - auto* pathInfo = GetPathInfoOptional(pathId); - if (!pathInfo) { - return false; - } else { - return (ui64)pathInfo->GetCommittedSize() > TCompactionLimits::OVERLOAD_INSERT_TABLE_SIZE_BY_PATH_ID; - } -} - -void TInsertionSummary::OnNewInserted(TPathInfo& pathInfo, const ui64 dataSize, const bool load) noexcept { - Counters.Inserted.Add(dataSize, load); - pathInfo.AddInsertedSize(dataSize, TCompactionLimits::OVERLOAD_INSERT_TABLE_SIZE_BY_PATH_ID); - ++StatsPrepared.Rows; - StatsPrepared.Bytes += dataSize; - AFL_VERIFY(Counters.Inserted.GetDataSize() == (i64)StatsPrepared.Bytes); -} - -void TInsertionSummary::OnEraseInserted(TPathInfo& pathInfo, const ui64 dataSize) noexcept { - Counters.Inserted.Erase(dataSize); - pathInfo.AddInsertedSize(-1 * (i64)dataSize, TCompactionLimits::OVERLOAD_INSERT_TABLE_SIZE_BY_PATH_ID); - Y_ABORT_UNLESS(--StatsPrepared.Rows >= 0); - Y_ABORT_UNLESS(StatsPrepared.Bytes >= dataSize); - StatsPrepared.Bytes -= dataSize; - AFL_VERIFY(Counters.Inserted.GetDataSize() == (i64)StatsPrepared.Bytes); -} - -THashSet TInsertionSummary::GetExpiredInsertions(const TInstant timeBorder, const ui64 limit) const { - return Inserted.GetExpired(timeBorder, limit); -} - -bool TInsertionSummary::EraseAborted(const TInsertWriteId writeId) { - auto it = Aborted.find(writeId); - if (it == Aborted.end()) { - return false; - } - Counters.Aborted.Erase(it->second.BlobSize()); - Aborted.erase(it); - return true; -} - -bool TInsertionSummary::HasAborted(const TInsertWriteId writeId) { - auto it = Aborted.find(writeId); - if (it == Aborted.end()) { - return false; - } - return true; -} - -bool TInsertionSummary::EraseCommitted(const TCommittedData& data) { - TPathInfo* pathInfo = GetPathInfoOptional(data.GetPathId()); - if (!pathInfo) { - Counters.Committed.SkipErase(data.BlobSize()); - return false; - } - - if (!pathInfo->EraseCommitted(data)) { - Counters.Committed.SkipErase(data.BlobSize()); - return false; - } else { - return true; - } -} - -bool TInsertionSummary::HasCommitted(const TCommittedData& data) { - TPathInfo* pathInfo = GetPathInfoOptional(data.GetPathId()); - if (!pathInfo) { - return false; - } - return pathInfo->HasCommitted(data); -} - -const TInsertedData* TInsertionSummary::AddAborted(TInsertedData&& data, const bool load /*= false*/) { - const TInsertWriteId writeId = data.GetInsertWriteId(); - Counters.Aborted.Add(data.BlobSize(), load); - AFL_VERIFY_DEBUG(!Inserted.contains(writeId)); - auto insertInfo = Aborted.emplace(writeId, std::move(data)); - AFL_VERIFY(insertInfo.second)("write_id", writeId); - return &insertInfo.first->second; -} - -std::optional TInsertionSummary::ExtractInserted(const TInsertWriteId id) { - auto result = Inserted.ExtractOptional(id); - if (result) { - auto pathInfo = GetPathInfoOptional(result->GetPathId()); - if (pathInfo) { - OnEraseInserted(*pathInfo, result->BlobSize()); - } - } - return result; -} - -const TInsertedData* TInsertionSummary::AddInserted(TInsertedData&& data, const bool load /*= false*/) { - auto* insertInfo = Inserted.AddVerified(std::move(data)); - AFL_VERIFY_DEBUG(!Aborted.contains(insertInfo->GetInsertWriteId())); - OnNewInserted(GetPathInfoVerified(insertInfo->GetPathId()), insertInfo->BlobSize(), load); - return insertInfo; -} - -} diff --git a/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.h b/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.h deleted file mode 100644 index 87af718db336..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/rt_insertion.h +++ /dev/null @@ -1,228 +0,0 @@ -#pragma once -#include "inserted.h" -#include "path_info.h" - -#include -#include - -#include - -namespace NKikimr::NOlap { -class IBlobsDeclareRemovingAction; - -class TInsertedDataInstant { -private: - const TInsertedData* Data; - const TInstant WriteTime; - -public: - TInsertedDataInstant(const TInsertedData& data) - : Data(&data) - , WriteTime(Data->GetMeta().GetDirtyWriteTime()) - { - - } - - const TInsertedData& GetData() const { - return *Data; - } - TInstant GetWriteTime() const { - return WriteTime; - } - - bool operator<(const TInsertedDataInstant& item) const { - if (WriteTime == item.WriteTime) { - return Data->GetInsertWriteId() < item.Data->GetInsertWriteId(); - } else { - return WriteTime < item.WriteTime; - } - } -}; - -class TInsertedContainer { -private: - THashMap Inserted; - std::set InsertedByWriteTime; - -public: - size_t size() const { - return Inserted.size(); - } - - bool contains(const TInsertWriteId id) const { - return Inserted.contains(id); - } - - THashMap::const_iterator begin() const { - return Inserted.begin(); - } - - THashMap::const_iterator end() const { - return Inserted.end(); - } - - THashSet GetExpired(const TInstant timeBorder, const ui64 limit) const { - THashSet result; - for (auto& data : InsertedByWriteTime) { - if (timeBorder < data.GetWriteTime()) { - break; - } - if (data.GetData().IsNotAbortable()) { - continue; - } - result.emplace(data.GetData().GetInsertWriteId()); - if (limit <= result.size()) { - break; - } - } - return result; - } - - TInsertedData* AddVerified(TInsertedData&& data) { - const TInsertWriteId writeId = data.GetInsertWriteId(); - auto itInsertion = Inserted.emplace(writeId, std::move(data)); - AFL_VERIFY(itInsertion.second); - auto* dataPtr = &itInsertion.first->second; - InsertedByWriteTime.emplace(TInsertedDataInstant(*dataPtr)); - return dataPtr; - } - - const TInsertedData* GetOptional(const TInsertWriteId id) const { - auto it = Inserted.find(id); - if (it == Inserted.end()) { - return nullptr; - } else { - return &it->second; - } - } - - TInsertedData* MutableOptional(const TInsertWriteId id) { - auto it = Inserted.find(id); - if (it == Inserted.end()) { - return nullptr; - } else { - return &it->second; - } - } - - std::optional ExtractOptional(const TInsertWriteId id) { - auto it = Inserted.find(id); - if (it == Inserted.end()) { - return std::nullopt; - } - AFL_VERIFY(InsertedByWriteTime.erase(TInsertedDataInstant(it->second))); - TInsertedData result = std::move(it->second); - Inserted.erase(it); - return result; - } -}; - -class TInsertionSummary { -public: - struct TCounters { - ui64 Rows{}; - ui64 Bytes{}; - ui64 RawBytes{}; - }; - -private: - friend class TPathInfo; - TCounters StatsPrepared; - TCounters StatsCommitted; - const NColumnShard::TInsertTableCounters Counters; - - TInsertedContainer Inserted; - THashMap Aborted; - - std::map> Priorities; - THashMap PathInfo; - void RemovePriority(const TPathInfo& pathInfo) noexcept; - void AddPriority(const TPathInfo& pathInfo) noexcept; - - void OnNewCommitted(const ui64 dataSize, const bool load = false) noexcept; - void OnEraseCommitted(TPathInfo& pathInfo, const ui64 dataSize) noexcept; - void OnNewInserted(TPathInfo& pathInfo, const ui64 dataSize, const bool load) noexcept; - void OnEraseInserted(TPathInfo& pathInfo, const ui64 dataSize) noexcept; - static TAtomicCounter CriticalInserted; - -public: - bool HasPathIdData(const TInternalPathId pathId) const { - auto it = PathInfo.find(pathId); - if (it == PathInfo.end()) { - return false; - } - return !it->second.IsEmpty(); - } - - void ErasePath(const TInternalPathId pathId) { - auto it = PathInfo.find(pathId); - if (it == PathInfo.end()) { - return; - } - RemovePriority(it->second); - AFL_VERIFY(it->second.IsEmpty()); - PathInfo.erase(it); - } - - void MarkAsNotAbortable(const TInsertWriteId writeId) { - auto* data = Inserted.MutableOptional(writeId); - if (!data) { - return; - } - data->MarkAsNotAbortable(); - } - - THashSet GetExpiredInsertions(const TInstant timeBorder, const ui64 limit) const; - - const TInsertedContainer& GetInserted() const { - return Inserted; - } - const THashMap& GetAborted() const { - return Aborted; - } - - const TInsertedData* AddAborted(TInsertedData&& data, const bool load = false); - bool EraseAborted(const TInsertWriteId writeId); - bool HasAborted(const TInsertWriteId writeId); - - bool EraseCommitted(const TCommittedData& data); - bool HasCommitted(const TCommittedData& data); - - const TInsertedData* AddInserted(TInsertedData&& data, const bool load = false); - std::optional ExtractInserted(const TInsertWriteId id); - - const TCounters& GetCountersPrepared() const { - return StatsPrepared; - } - const TCounters& GetCountersCommitted() const { - return StatsCommitted; - } - const NColumnShard::TInsertTableCounters& GetCounters() const { - return Counters; - } - NKikimr::NOlap::TPathInfo& RegisterPathInfo(const TInternalPathId pathId); - TPathInfo* GetPathInfoOptional(const TInternalPathId pathId); - const TPathInfo* GetPathInfoOptional(const TInternalPathId pathId) const; - TPathInfo& GetPathInfoVerified(const TInternalPathId pathId) { - auto* result = GetPathInfoOptional(pathId); - AFL_VERIFY(result); - return *result; - } - const TPathInfo& GetPathInfoVerified(const TInternalPathId pathId) const { - auto* result = GetPathInfoOptional(pathId); - AFL_VERIFY(result); - return *result; - } - - const THashMap& GetPathInfo() const { - return PathInfo; - } - - bool IsOverloaded(const TInternalPathId pathId) const; - - const std::map>& GetPathPriorities() const { - return Priorities; - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/user_data.cpp b/ydb/core/tx/columnshard/engines/insert_table/user_data.cpp deleted file mode 100644 index c0f6ff4d54da..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/user_data.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "user_data.h" -#include - -namespace NKikimr::NOlap { - -namespace { - -class TInsertTableCacheController { -private: - TAtomicCounter BlobsCacheSize = 0; - const i64 BlobsCacheLimit = (i64)1 << 30; -public: - void Return(const ui64 size) { - const i64 val = BlobsCacheSize.Sub(size); - AFL_VERIFY(val >= 0)("size", size)("val", val); - } - - bool Take(const ui64 size) { - if (BlobsCacheSize.Add(size) <= BlobsCacheLimit) { - return true; - } - const i64 val = BlobsCacheSize.Sub(size); - AFL_VERIFY(val >= 0)("size", size)("val", val); - return false; - } -}; - -} - -TUserData::TBlobStorageGuard::~TBlobStorageGuard() { - Singleton()->Return(Data.size()); -} - - TUserData::TUserData(const TInternalPathId pathId, const TBlobRange& blobRange, const NKikimrTxColumnShard::TLogicalMetadata& proto, - const ui64 schemaVersion, const std::optional& blobData) - : Meta(proto) - , BlobRange(blobRange) - , PathId(pathId) - , SchemaVersion(schemaVersion) { - if (blobData && Singleton()->Take(blobData->size())) { - BlobDataGuard = std::make_shared(*blobData); - } -} - -} diff --git a/ydb/core/tx/columnshard/engines/insert_table/user_data.h b/ydb/core/tx/columnshard/engines/insert_table/user_data.h deleted file mode 100644 index 9a6e2873c130..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/user_data.h +++ /dev/null @@ -1,101 +0,0 @@ -#pragma once -#include "meta.h" - -#include - -#include -#include - -namespace NKikimr::NOlap { - -class TUserData { -private: - TInsertedDataMeta Meta; - YDB_READONLY_DEF(TBlobRange, BlobRange); - class TBlobStorageGuard { - private: - YDB_READONLY_DEF(TString, Data); - - public: - TBlobStorageGuard(const TString& data) - : Data(data) { - } - ~TBlobStorageGuard(); - }; - - std::shared_ptr BlobDataGuard; - YDB_READONLY_DEF(TInternalPathId, PathId); - YDB_READONLY(ui64, SchemaVersion, 0); - -public: - TUserData() = delete; - TUserData(const TInternalPathId pathId, const TBlobRange& blobRange, const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, - const std::optional& blobData); - - static std::shared_ptr Build(const TInternalPathId pathId, const TBlobRange& blobRange, const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, - const std::optional& blobData) { - return std::make_shared(pathId, blobRange, proto, schemaVersion, blobData); - } - - static std::shared_ptr Build(const TInternalPathId pathId, const TUnifiedBlobId& blobId, const NKikimrTxColumnShard::TLogicalMetadata& proto, const ui64 schemaVersion, - const std::optional& blobData) { - return std::make_shared(pathId, TBlobRange(blobId), proto, schemaVersion, blobData); - } - - std::optional GetBlobData() const { - if (BlobDataGuard) { - return BlobDataGuard->GetData(); - } else { - return std::nullopt; - } - } - - ui64 GetTxVolume() const { - return Meta.GetTxVolume() + sizeof(TBlobRange); - } - - const TInsertedDataMeta& GetMeta() const { - return Meta; - } -}; - -class TUserDataContainer { -protected: - std::shared_ptr UserData; - -public: - TUserDataContainer(const std::shared_ptr& userData) - : UserData(userData) { - AFL_VERIFY(UserData); - } - - ui64 GetSchemaVersion() const { - return UserData->GetSchemaVersion(); - } - - ui32 BlobSize() const { - return GetBlobRange().Size; - } - - ui32 GetTxVolume() const { - return UserData->GetTxVolume(); - } - - TInternalPathId GetPathId() const { - return UserData->GetPathId(); - } - - const TBlobRange& GetBlobRange() const { - return UserData->GetBlobRange(); - } - - std::optional GetBlobData() const { - return UserData->GetBlobData(); - } - - const TInsertedDataMeta& GetMeta() const { - return UserData->GetMeta(); - } -}; - -} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/insert_table/ya.make b/ydb/core/tx/columnshard/engines/insert_table/ya.make deleted file mode 100644 index e6fde75077d5..000000000000 --- a/ydb/core/tx/columnshard/engines/insert_table/ya.make +++ /dev/null @@ -1,21 +0,0 @@ -LIBRARY() - -SRCS( - insert_table.cpp - rt_insertion.cpp - user_data.cpp - inserted.cpp - committed.cpp - path_info.cpp - meta.cpp -) - -PEERDIR( - contrib/libs/apache/arrow - ydb/library/formats/arrow/modifier - ydb/core/protos - ydb/core/formats/arrow - ydb/core/tablet_flat -) - -END() diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp index 58896e6d529e..671e343ba732 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.cpp @@ -4,9 +4,8 @@ namespace NKikimr::NOlap::NReader { -TDataStorageAccessor::TDataStorageAccessor(const std::unique_ptr& insertTable, const std::unique_ptr& index) - : InsertTable(insertTable) - , Index(index) { +TDataStorageAccessor::TDataStorageAccessor(const std::unique_ptr& index) + : Index(index) { } std::shared_ptr TDataStorageAccessor::Select(const TReadDescription& readDescription, const bool withUncommitted) const { @@ -24,11 +23,4 @@ ISnapshotSchema::TPtr TReadMetadataBase::GetLoadSchemaVerified(const TPortionInf return schema; } -std::vector TDataStorageAccessor::GetCommitedBlobs(const TReadDescription& readDescription, - const std::shared_ptr& pkSchema, const std::optional lockId, const TSnapshot& reqSnapshot) const { - AFL_VERIFY(readDescription.PKRangesFilter); - return std::move(InsertTable->Read(readDescription.PathId.InternalPathId, lockId, reqSnapshot, - pkSchema, &*readDescription.PKRangesFilter)); -} - } // namespace NKikimr::NOlap::NReader diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h index b369cec47373..acd53c064bbb 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include #include @@ -20,14 +19,11 @@ class TReadContext; class TDataStorageAccessor { private: - const std::unique_ptr& InsertTable; const std::unique_ptr& Index; public: - TDataStorageAccessor(const std::unique_ptr& insertTable, const std::unique_ptr& index); + TDataStorageAccessor(const std::unique_ptr& index); std::shared_ptr Select(const TReadDescription& readDescription, const bool withUncommitted) const; - std::vector GetCommitedBlobs(const TReadDescription& readDescription, const std::shared_ptr& pkSchema, - const std::optional lockId, const TSnapshot& reqSnapshot) const; }; // Holds all metadata that is needed to perform read/scan diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/ya.make b/ydb/core/tx/columnshard/engines/reader/abstract/ya.make index bf15ccad4a4b..28da043e01cb 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/ya.make +++ b/ydb/core/tx/columnshard/engines/reader/abstract/ya.make @@ -9,7 +9,6 @@ SRCS( PEERDIR( ydb/core/tx/columnshard/engines/scheme/versions - ydb/core/tx/columnshard/engines/insert_table ydb/core/tx/program ydb/core/protos ydb/core/tx/columnshard/data_sharing/protos diff --git a/ydb/core/tx/columnshard/engines/reader/common/stats.cpp b/ydb/core/tx/columnshard/engines/reader/common/stats.cpp index 36be742dd0f6..5a9c72d4dd6b 100644 --- a/ydb/core/tx/columnshard/engines/reader/common/stats.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common/stats.cpp @@ -11,7 +11,6 @@ void TReadStats::PrintToLog() { ("index_granules", IndexGranules) ("index_portions", IndexPortions) ("index_batches", IndexBatches) - ("committed_batches", CommittedBatches) ("schema_columns", SchemaColumns) ("filter_columns", FilterColumns) ("additional_columns", AdditionalColumns) diff --git a/ydb/core/tx/columnshard/engines/reader/common/stats.h b/ydb/core/tx/columnshard/engines/reader/common/stats.h index 3feb8459cee0..02a19b043186 100644 --- a/ydb/core/tx/columnshard/engines/reader/common/stats.h +++ b/ydb/core/tx/columnshard/engines/reader/common/stats.h @@ -9,7 +9,6 @@ struct TReadStats { ui64 IndexGranules{0}; ui64 IndexPortions{0}; ui64 IndexBatches{0}; - ui64 CommittedBatches{0}; ui64 CommittedPortionsBytes = 0; ui64 InsertedPortionsBytes = 0; ui64 CompactedPortionsBytes = 0; diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp index 89a07f35b282..0f223dbde9ce 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp @@ -24,11 +24,8 @@ TConclusionStatus TReadMetadata::Init( if (!i->IsCommitted()) { AFL_VERIFY(i->GetPortionType() == EPortionType::Written); auto* written = static_cast(i.get()); - if (owner->HasLongTxWrites(written->GetInsertWriteId())) { - } else { - auto op = owner->GetOperationsManager().GetOperationByInsertWriteIdVerified(written->GetInsertWriteId()); - AddWriteIdToCheck(written->GetInsertWriteId(), op->GetLockId()); - } + auto op = owner->GetOperationsManager().GetOperationByInsertWriteIdVerified(written->GetInsertWriteId()); + AddWriteIdToCheck(written->GetInsertWriteId(), op->GetLockId()); } } } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/constructor.cpp index ee01720c604c..36401bacbe92 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/constructor.cpp @@ -22,9 +22,8 @@ std::vector TIndexScannerConstructor::GetPrimaryKeyScheme(const N NKikimr::TConclusion> TIndexScannerConstructor::DoBuildReadMetadata( const NColumnShard::TColumnShard* self, const TReadDescription& read) const { - auto& insertTable = self->InsertTable; auto& index = self->TablesManager.GetPrimaryIndex(); - if (!insertTable || !index) { + if (!index) { return std::shared_ptr(); } @@ -33,7 +32,7 @@ NKikimr::TConclusion> TIndexScannerConstructo << self->GetMinReadSnapshot() << ". now: " << TInstant::Now()); } - TDataStorageAccessor dataAccessor(insertTable, index); + TDataStorageAccessor dataAccessor(index); AFL_VERIFY(read.PathId); auto readCopy = read; if (readCopy.GetSorting() == ERequestSorting::NONE) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp index 9d1bbaac2d73..7e68da1d4e8f 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.cpp @@ -11,21 +11,7 @@ std::unique_ptr TReadMetadata::StartScan(const std::shared_pt } TConclusionStatus TReadMetadata::DoInitCustom( - const NColumnShard::TColumnShard* owner, const TReadDescription& readDescription, const TDataStorageAccessor& dataAccessor) { - CommittedBlobs = - dataAccessor.GetCommitedBlobs(readDescription, ResultIndexSchema->GetIndexInfo().GetReplaceKey(), LockId, GetRequestSnapshot()); - - if (LockId) { - for (auto&& i : CommittedBlobs) { - if (!i.IsCommitted()) { - if (owner->HasLongTxWrites(i.GetInsertWriteId())) { - } else { - auto op = owner->GetOperationsManager().GetOperationByInsertWriteIdVerified(i.GetInsertWriteId()); - AddWriteIdToCheck(i.GetInsertWriteId(), op->GetLockId()); - } - } - } - } + const NColumnShard::TColumnShard* /*owner*/, const TReadDescription& /*readDescription*/, const TDataStorageAccessor& /*dataAccessor*/) { return TConclusionStatus::Success(); } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.h index b0242d486aaa..9a3928fda543 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/constructor/read_metadata.h @@ -18,17 +18,16 @@ class TReadMetadata: public NCommon::TReadMetadata { using TConstPtr = std::shared_ptr; using TBase::TBase; - std::vector CommittedBlobs; virtual bool Empty() const override { Y_ABORT_UNLESS(SelectInfo); - return SelectInfo->Portions.empty() && CommittedBlobs.empty(); + return SelectInfo->Portions.empty(); } virtual std::shared_ptr BuildReader(const std::shared_ptr& context) const override; virtual std::unique_ptr StartScan(const std::shared_ptr& readContext) const override; virtual TString DebugString() const override { - return TBase::DebugString() + ";committed=" + ::ToString(CommittedBlobs.size()); + return TBase::DebugString(); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp index 2e537f55a2c2..c54e7fc44e5c 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/plain_read_data.cpp @@ -11,52 +11,28 @@ TPlainReadData::TPlainReadData(const std::shared_ptr& context) std::deque> sources; const auto readMetadata = GetReadMetadataVerifiedAs(); const auto& portions = GetReadMetadata()->SelectInfo->Portions; - const auto& committed = readMetadata->CommittedBlobs; ui64 compactedPortionsBytes = 0; ui64 insertedPortionsBytes = 0; ui64 committedPortionsBytes = 0; for (auto&& i : portions) { if (i->GetPortionType() == EPortionType::Compacted) { compactedPortionsBytes += i->GetTotalBlobBytes(); - } else { + } else if (i->GetProduced() == NPortion::EProduced::INSERTED) { insertedPortionsBytes += i->GetTotalBlobBytes(); + } else { + committedPortionsBytes += i->GetTotalBlobBytes(); } sources.emplace_back(std::make_shared(sourceIdx++, i, SpecialReadContext)); } - for (auto&& i : committed) { - if (i.IsCommitted()) { - continue; - } - if (GetReadMetadata()->IsMyUncommitted(i.GetInsertWriteId())) { - continue; - } - if (GetReadMetadata()->GetPKRangesFilter().CheckPoint(i.GetFirst()) || - GetReadMetadata()->GetPKRangesFilter().CheckPoint(i.GetLast())) { - GetReadMetadata()->SetConflictedWriteId(i.GetInsertWriteId()); - } - } - - for (auto&& i : committed) { - if (!i.IsCommitted()) { - if (GetReadMetadata()->IsWriteConflictable(i.GetInsertWriteId())) { - continue; - } - } else if (!GetReadMetadata()->GetPKRangesFilter().IsUsed(i.GetFirst(), i.GetLast())) { - continue; - } - sources.emplace_back(std::make_shared(sourceIdx++, i, SpecialReadContext)); - committedPortionsBytes += i.GetSize(); - } Scanner = std::make_shared(std::move(sources), SpecialReadContext); auto& stats = GetReadMetadata()->ReadStats; stats->IndexPortions = GetReadMetadata()->SelectInfo->Portions.size(); stats->IndexBatches = GetReadMetadata()->NumIndexedBlobs(); - stats->CommittedBatches = readMetadata->CommittedBlobs.size(); stats->SchemaColumns = (*SpecialReadContext->GetProgramInputColumns() - *SpecialReadContext->GetSpecColumns()).GetColumnsCount(); - stats->CommittedPortionsBytes = committedPortionsBytes; stats->InsertedPortionsBytes = insertedPortionsBytes; stats->CompactedPortionsBytes = compactedPortionsBytes; + stats->CommittedPortionsBytes = committedPortionsBytes; } std::vector> TPlainReadData::DoExtractReadyResults(const int64_t /*maxRowsInBatch*/) { diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index e09c63ef9988..664a10159c40 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -212,60 +212,4 @@ bool TPortionDataSource::DoAddTxConflict() { return false; } -bool TCommittedDataSource::DoStartFetchingColumns( - const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& /*columns*/) { - if (ReadStarted) { - return false; - } - ReadStarted = true; - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", step.GetName())("fetching_info", step.DebugString()); - - std::shared_ptr storageOperator = GetContext()->GetCommonContext()->GetStoragesManager()->GetInsertOperator(); - auto readAction = storageOperator->StartReadingAction(NBlobOperations::EConsumer::SCAN); - - readAction->SetIsBackgroundProcess(false); - readAction->AddRange(CommittedBlob.GetBlobRange()); - - std::vector> actions = { readAction }; - auto constructor = std::make_shared(actions, sourcePtr, step, GetContext(), "CS::READ::" + step.GetName(), ""); - NActors::TActivationContext::AsActorContext().Register(new NOlap::NBlobOperations::NRead::TActor(constructor)); - return true; -} - -void TCommittedDataSource::DoAssembleColumns(const std::shared_ptr& columns, const bool /*sequential*/) { - TMemoryProfileGuard mGuard("SCAN_PROFILE::ASSEMBLER::COMMITTED", IS_DEBUG_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD_SCAN_MEMORY)); - const ISnapshotSchema::TPtr batchSchema = - GetContext()->GetReadMetadata()->GetIndexVersions().GetSchemaVerified(GetCommitted().GetSchemaVersion()); - const ISnapshotSchema::TPtr resultSchema = GetContext()->GetReadMetadata()->GetResultSchema(); - if (!AssembledFlag) { - AssembledFlag = true; - AFL_VERIFY(GetStageData().GetBlobs().size() == 1); - auto bData = MutableStageData().ExtractBlob(GetStageData().GetBlobs().begin()->first); - auto schema = GetContext()->GetReadMetadata()->GetBlobSchema(CommittedBlob.GetSchemaVersion()); - auto rBatch = NArrow::DeserializeBatch( - bData, std::make_shared(CommittedBlob.GetSchemaSubset().Apply(schema.begin(), schema.end()))); - AFL_VERIFY(rBatch)("schema", schema.ToString()); - auto batch = std::make_shared(rBatch); - std::set columnIdsToDelete = batchSchema->GetColumnIdsToDelete(resultSchema); - if (!columnIdsToDelete.empty()) { - batch->DeleteFieldsByIndex(batchSchema->ConvertColumnIdsToIndexes(columnIdsToDelete)); - } - TSnapshot ss = TSnapshot::Zero(); - if (CommittedBlob.IsCommitted()) { - ss = CommittedBlob.GetCommittedSnapshotVerified(); - } else { - ss = GetContext()->GetReadMetadata()->IsMyUncommitted(CommittedBlob.GetInsertWriteId()) - ? GetContext()->GetReadMetadata()->GetRequestSnapshot() - : TSnapshot::Zero(); - } - GetContext()->GetReadMetadata()->GetIndexInfo().AddSnapshotColumns(*batch, ss, (ui64)CommittedBlob.GetInsertWriteId()); - GetContext()->GetReadMetadata()->GetIndexInfo().AddDeleteFlagsColumn(*batch, CommittedBlob.GetIsDelete()); - MutableStageData().AddBatch(batch, *GetContext()->GetCommonContext()->GetResolver(), true); - if (CommittedBlob.GetIsDelete()) { - MutableStageData().AddFilter(NArrow::TColumnFilter::BuildDenyFilter()); - } - } - MutableStageData().SyncTableColumns(columns->GetSchema()->fields(), *resultSchema, GetRecordsCount()); -} - } // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h index 29bb9e4a7cd3..77f4d5d533c4 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.h @@ -292,107 +292,4 @@ class TPortionDataSource: public IDataSource { } }; -class TCommittedDataSource: public IDataSource { -private: - using TBase = IDataSource; - TCommittedBlob CommittedBlob; - bool ReadStarted = false; - bool AssembledFlag = false; - - virtual void DoAbort() override { - } - - virtual bool DoStartFetchingColumns( - const std::shared_ptr& sourcePtr, const TFetchingScriptCursor& step, const TColumnsSetIds& columns) override; - - virtual void DoAssembleColumns(const std::shared_ptr& columns, const bool sequential) override; - virtual NJson::TJsonValue DoDebugJson() const override { - NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("type", "commit"); - result.InsertValue("info", CommittedBlob.DebugString()); - return result; - } - virtual TInternalPathId GetPathId() const override { - return TInternalPathId{}; - } - - virtual bool DoAddTxConflict() override { - if (CommittedBlob.IsCommitted()) { - GetContext()->GetReadMetadata()->SetBrokenWithCommitted(); - return true; - } else if (!GetContext()->GetReadMetadata()->IsMyUncommitted(CommittedBlob.GetInsertWriteId())) { - GetContext()->GetReadMetadata()->SetConflictedWriteId(CommittedBlob.GetInsertWriteId()); - return true; - } - return false; - } - -public: - virtual ui64 PredictAccessorsMemory() const override { - return 0; - } - - virtual bool NeedAccessorsForRead() const override { - return false; - } - - virtual bool NeedAccessorsFetching() const override { - return false; - } - - virtual THashMap DecodeBlobAddresses(NBlobOperations::NRead::TCompositeReadBlobs&& blobsOriginal) const override { - THashMap result; - for (auto&& i : blobsOriginal) { - for (auto&& b : i.second) { - result.emplace(TChunkAddress(1, 1), std::move(b.second)); - } - } - return result; - } - - virtual bool HasIndexes(const std::set& /*indexIds*/) const override { - return false; - } - - virtual ui64 GetColumnRawBytes(const std::set& /*columnIds*/) const override { - return CommittedBlob.GetBlobRange().Size; - } - - virtual ui64 GetColumnBlobBytes(const std::set& /*columnsIds*/) const override { - return CommittedBlob.GetBlobRange().Size; - } - - virtual bool DoStartFetchingAccessor(const std::shared_ptr& /*sourcePtr*/, const TFetchingScriptCursor& /*step*/) override { - return false; - } - - virtual ui64 GetColumnsVolume(const std::set& columnIds, const EMemType type) const override { - AFL_VERIFY(columnIds.size()); - switch (type) { - case EMemType::Raw: - return GetColumnRawBytes(columnIds); - case EMemType::Blob: - return GetColumnBlobBytes(columnIds); - case EMemType::RawSequential: - return GetColumnRawBytes(columnIds); - } - } - - virtual ui64 GetIndexRawBytes(const std::set& /*columnIds*/) const override { - AFL_VERIFY(false); - return 0; - } - - const TCommittedBlob& GetCommitted() const { - return CommittedBlob; - } - - TCommittedDataSource(const ui32 sourceIdx, const TCommittedBlob& committed, const std::shared_ptr& context) - : TBase((ui64)committed.GetInsertWriteId(), sourceIdx, context, committed.GetFirst(), committed.GetLast(), - committed.GetCommittedSnapshotDef(TSnapshot::Zero()), committed.GetCommittedSnapshotDef(TSnapshot::Zero()), - committed.GetRecordsCount(), {}, committed.GetIsDelete()) - , CommittedBlob(committed) { - } -}; - } // namespace NKikimr::NOlap::NReader::NPlain diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/constructor/constructor.cpp index 0363e2385c33..6fe898757e7a 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/constructor/constructor.cpp @@ -21,9 +21,8 @@ std::vector TIndexScannerConstructor::GetPrimaryKeyScheme(const N NKikimr::TConclusion> TIndexScannerConstructor::DoBuildReadMetadata( const NColumnShard::TColumnShard* self, const TReadDescription& read) const { - auto& insertTable = self->InsertTable; auto& index = self->TablesManager.GetPrimaryIndex(); - if (!insertTable || !index) { + if (!index) { return std::shared_ptr(); } @@ -32,7 +31,7 @@ NKikimr::TConclusion> TIndexScannerConstructo << self->GetMinReadSnapshot() << ". now: " << TInstant::Now()); } - TDataStorageAccessor dataAccessor(insertTable, index); + TDataStorageAccessor dataAccessor(index); AFL_VERIFY(read.PathId); auto readMetadata = std::make_shared(index->CopyVersionedIndexPtr(), read); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp index 1d6f546e978f..d65ddac0f7d0 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/plain_read_data.cpp @@ -12,11 +12,14 @@ TPlainReadData::TPlainReadData(const std::shared_ptr& context) const auto& portions = GetReadMetadata()->SelectInfo->Portions; ui64 compactedPortionsBytes = 0; ui64 insertedPortionsBytes = 0; + ui64 committedPortionsBytes = 0; for (auto&& i : portions) { if (i->GetPortionType() == EPortionType::Compacted) { compactedPortionsBytes += i->GetTotalBlobBytes(); - } else { + } else if (i->GetProduced() == NPortion::EProduced::INSERTED) { insertedPortionsBytes += i->GetTotalBlobBytes(); + } else { + committedPortionsBytes += i->GetTotalBlobBytes(); } sources.emplace_back(TSourceConstructor(sourceIdx++, i, context)); @@ -29,6 +32,7 @@ TPlainReadData::TPlainReadData(const std::shared_ptr& context) stats->SchemaColumns = (*SpecialReadContext->GetProgramInputColumns() - *SpecialReadContext->GetSpecColumns()).GetColumnsCount(); stats->InsertedPortionsBytes = insertedPortionsBytes; stats->CompactedPortionsBytes = compactedPortionsBytes; + stats->CommittedPortionsBytes = committedPortionsBytes; } std::vector> TPlainReadData::DoExtractReadyResults(const int64_t /*maxRowsInBatch*/) { diff --git a/ydb/core/tx/columnshard/engines/ut/helper.cpp b/ydb/core/tx/columnshard/engines/ut/helper.cpp index 56a5c26ba492..8f2445a8ced3 100644 --- a/ydb/core/tx/columnshard/engines/ut/helper.cpp +++ b/ydb/core/tx/columnshard/engines/ut/helper.cpp @@ -1,7 +1,11 @@ #include "helper.h" + +#include + +#include #include -namespace NKikimr::NOlap::NEngines::NTest { +namespace NKikimr::NOlap::NEngines::NTest { std::shared_ptr TLocalHelper::GetMetaSchema() { return std::make_shared(arrow::FieldVector({ std::make_shared("1", arrow::uint64()) })); @@ -23,4 +27,4 @@ NKikimrTxColumnShard::TLogicalMetadata TLocalHelper::GetMetaProto() { return result; } -} \ No newline at end of file +} // namespace NKikimr::NOlap::NEngines::NTest diff --git a/ydb/core/tx/columnshard/engines/ut/helper.h b/ydb/core/tx/columnshard/engines/ut/helper.h index c7072c13af91..f768b7877eb1 100644 --- a/ydb/core/tx/columnshard/engines/ut/helper.h +++ b/ydb/core/tx/columnshard/engines/ut/helper.h @@ -1,5 +1,7 @@ #pragma once -#include +#include + +#include namespace NKikimr::NOlap::NEngines::NTest { @@ -9,4 +11,4 @@ class TLocalHelper { static std::shared_ptr GetMetaSchema(); }; -}; \ No newline at end of file +}; // namespace NKikimr::NOlap::NEngines::NTest diff --git a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp deleted file mode 100644 index 83b18e659438..000000000000 --- a/ydb/core/tx/columnshard/engines/ut/ut_insert_table.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "helper.h" - -#include -#include -#include - -#include -#include - -namespace NKikimr { - -using namespace NOlap; -using namespace NKikimr::NOlap::NEngines::NTest; - -namespace { - -class TTestInsertTableDB : public IDbWrapper { -public: - virtual const IBlobGroupSelector* GetDsGroupSelector() const override { - return &Default(); - } - - virtual void WriteColumns(const NOlap::TPortionInfo& /*portion*/, const NKikimrTxColumnShard::TIndexPortionAccessor& /*proto*/) override { - - } - void Insert(const TInsertedData&) override { - } - void Commit(const TCommittedData&) override { - } - void Abort(const TInsertedData&) override { - } - void EraseInserted(const TInsertedData&) override { - } - void EraseCommitted(const TCommittedData&) override { - } - void EraseAborted(const TInsertedData&) override { - } - - virtual TConclusion>> LoadGranulesShardingInfo() override { - THashMap> result; - return result; - } - - bool Load(TInsertTableAccessor&, const TInstant&) override { - return true; - } - - virtual void WritePortion(const NOlap::TPortionInfo& /*portion*/) override { - } - virtual void ErasePortion(const NOlap::TPortionInfo& /*portion*/) override { - } - virtual bool LoadPortions(const std::optional /*reqPathId*/, - const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& /*callback*/) override { - return true; - } - - void WriteColumn(const TPortionInfo&, const TColumnRecord&, const ui32 /*firstPKColumnId*/) override { - } - void EraseColumn(const TPortionInfo&, const TColumnRecord&) override { - } - bool LoadColumns(const std::optional /*reqPathId*/, const std::function&) override { - return true; - } - - virtual void WriteIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { - } - virtual void EraseIndex(const TPortionInfo& /*portion*/, const TIndexChunk& /*row*/) override { - } - virtual bool LoadIndexes(const std::optional /*reqPathId*/, - const std::function& /*callback*/) override { - return true; - } - - void WriteCounter(ui32, ui64) override { - } - bool LoadCounters(const std::function&) override { - return true; - } -}; - -} // namespace - -Y_UNIT_TEST_SUITE(TColumnEngineTestInsertTable) { - Y_UNIT_TEST(TestInsertCommit) { - TInsertWriteId writeId = (TInsertWriteId)0; - const auto& tableId0 = TInternalPathId::FromRawValue(0); - const auto& tableId1 = TInternalPathId::FromRawValue(1); - TString dedupId = "0"; - TUnifiedBlobId blobId1(2222, 1, 1, 100, 2, 0, 1); - - TTestInsertTableDB dbTable; - TInsertTable insertTable; - ui64 indexSnapshot = 0; - - // insert, not commited - auto userData1 = std::make_shared(tableId0, TBlobRange(blobId1), TLocalHelper::GetMetaProto(), indexSnapshot, std::nullopt); - insertTable.RegisterPathInfo(tableId0); - bool ok = insertTable.Insert(dbTable, TInsertedData(writeId, userData1)); - UNIT_ASSERT(ok); - - // read nothing - auto blobs = insertTable.Read(tableId0, {}, TSnapshot::Zero(), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 0); - blobs = insertTable.Read(tableId1, {}, TSnapshot::Zero(), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 0); - - // commit - ui64 planStep = 100; - ui64 txId = 42; - insertTable.Commit(dbTable, planStep, txId, { writeId }, [](TInternalPathId) { - return true; - }); -// UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().size(), 1); -// UNIT_ASSERT_EQUAL(insertTable.GetPathPriorities().begin()->second.size(), 1); -// UNIT_ASSERT_EQUAL((*insertTable.GetPathPriorities().begin()->second.begin())->GetCommitted().size(), 1); - - // read old snapshot - blobs = insertTable.Read(tableId0, {}, TSnapshot::Zero(), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 0); - blobs = insertTable.Read(tableId1, {}, TSnapshot::Zero(), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 0); - - // read new snapshot - blobs = insertTable.Read(tableId0, {}, TSnapshot(planStep, txId), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 1); - blobs = insertTable.Read(tableId1, {}, TSnapshot::Zero(), TLocalHelper::GetMetaSchema(), nullptr); - UNIT_ASSERT_EQUAL(blobs.size(), 0); - } -} - -} // namespace NKikimr diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index fc85ba5ab143..20efcd84bc31 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -8,10 +8,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -63,46 +61,6 @@ class TTestDbWrapper: public IDbWrapper { return result; } - void Insert(const TInsertedData& data) override { - Inserted.emplace(data.GetInsertWriteId(), data); - } - - void Commit(const TCommittedData& data) override { - Committed[data.GetPathId()].emplace(data); - } - - void Abort(const TInsertedData& data) override { - Aborted.emplace(data.GetInsertWriteId(), data); - } - - void EraseInserted(const TInsertedData& data) override { - Inserted.erase(data.GetInsertWriteId()); - } - - void EraseCommitted(const TCommittedData& data) override { - Committed[data.GetPathId()].erase(data); - } - - void EraseAborted(const TInsertedData& data) override { - Aborted.erase(data.GetInsertWriteId()); - } - - bool Load(TInsertTableAccessor& accessor, const TInstant&) override { - for (auto&& i : Inserted) { - accessor.AddInserted(std::move(i.second), true); - } - for (auto&& i : Aborted) { - accessor.AddAborted(std::move(i.second), true); - } - for (auto&& i : Committed) { - for (auto&& c : i.second) { - auto copy = c; - accessor.AddCommitted(std::move(copy), true); - } - } - return true; - } - virtual void WritePortion(const NOlap::TPortionInfo& portion) override { auto it = Portions.find(portion.GetPortionId()); if (it == Portions.end()) { @@ -221,9 +179,6 @@ class TTestDbWrapper: public IDbWrapper { } private: - THashMap Inserted; - THashMap> Committed; - THashMap Aborted; THashMap> Portions; THashMap Indices; }; diff --git a/ydb/core/tx/columnshard/engines/ut/ya.make b/ydb/core/tx/columnshard/engines/ut/ya.make index 215cfe63763d..4b0cebad7090 100644 --- a/ydb/core/tx/columnshard/engines/ut/ya.make +++ b/ydb/core/tx/columnshard/engines/ut/ya.make @@ -31,8 +31,6 @@ ENDIF() YQL_LAST_ABI_VERSION() SRCS( - ut_insert_table.cpp - ut_logs_engine.cpp ut_program.cpp ut_script.cpp helper.cpp diff --git a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp index f6ebdc8b1426..dc76b1226494 100644 --- a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp +++ b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.cpp @@ -33,26 +33,6 @@ void TWideSerializedBatch::InitBlobId(const TUnifiedBlobId& id) { Range.BlobId = id; } -std::shared_ptr TWideSerializedBatch::BuildInsertionUserData(const NColumnShard::TColumnShard& owner) const { - NKikimrTxColumnShard::TLogicalMetadata meta; - meta.SetNumRows(SplittedBlobs.GetRowsCount()); - meta.SetRawBytes(SplittedBlobs.GetRawBytes()); - meta.SetDirtyWriteTimeSeconds(GetStartInstant().Seconds()); - meta.SetSpecialKeysRawData(SplittedBlobs.GetSpecialKeysFullSafe()); - meta.SetSpecialKeysPayloadData(SplittedBlobs.GetSpecialKeysPayloadSafe()); - - const auto& blobRange = Range; - Y_ABORT_UNLESS(blobRange.GetBlobId().IsValid()); - - const auto& writeMeta = GetAggregation().GetWriteMeta(); - meta.SetModificationType(TEnumOperator::SerializeToProto(writeMeta.GetModificationType())); - *meta.MutableSchemaSubset() = GetAggregation().GetSchemaSubset().SerializeToProto(); - auto schemeVersion = GetAggregation().GetSchemaVersion(); - auto tableSchema = owner.GetTablesManager().GetPrimaryIndex()->GetVersionedIndex().GetSchemaVerified(schemeVersion); - - return std::make_shared(writeMeta.GetPathId().InternalPathId, blobRange, meta, tableSchema->GetVersion(), SplittedBlobs.GetData()); -} - void TWritingBuffer::InitReadyInstant(const TMonotonic /*instant*/) { // for (auto&& aggr : Aggregations) { // aggr->MutableWriteMeta().SetWriteMiddle5StartInstant(instant); diff --git a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h index 1fcff6975285..b303e90b0e27 100644 --- a/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h +++ b/ydb/core/tx/columnshard/engines/writer/indexed_blob_constructor.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -28,7 +27,6 @@ class TWideSerializedBatch { TWriteAggregation* ParentAggregation; public: - std::shared_ptr BuildInsertionUserData(const NColumnShard::TColumnShard& owner) const; void InitBlobId(const TUnifiedBlobId& id); const NArrow::TSerializedBatch& GetSplittedBlobs() const { diff --git a/ydb/core/tx/columnshard/engines/ya.make b/ydb/core/tx/columnshard/engines/ya.make index c1feb421b952..587cc3c70e15 100644 --- a/ydb/core/tx/columnshard/engines/ya.make +++ b/ydb/core/tx/columnshard/engines/ya.make @@ -25,7 +25,6 @@ PEERDIR( ydb/core/tx/columnshard/engines/reader ydb/core/tx/columnshard/engines/predicate ydb/core/tx/columnshard/engines/storage - ydb/core/tx/columnshard/engines/insert_table ydb/core/tx/columnshard/engines/changes ydb/core/tx/columnshard/engines/portions ydb/core/tx/columnshard/engines/protos diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 52cd8b3fee2b..dbb538daff8e 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -65,7 +65,6 @@ class ILocalDBModifier { class ICSController { public: enum class EBackground { - Indexation, Compaction, TTL, Cleanup, diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index e4b720d2c71c..11c58e6bb194 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.cpp b/ydb/core/tx/columnshard/hooks/testing/ro_controller.cpp index 4803cfca501d..c4d109202ea7 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -31,9 +30,6 @@ bool TReadOnlyController::DoOnWriteIndexComplete(const NOlap::TColumnEngineChang if (change.TypeString() == NOlap::TTTLColumnEngineChanges::StaticTypeName()) { TTLFinishedCounter.Inc(); } - if (change.TypeString() == NOlap::TInsertColumnEngineChanges::StaticTypeName()) { - InsertFinishedCounter.Inc(); - } if (change.TypeString() == NOlap::TCompactColumnEngineChanges::StaticTypeName()) { CompactionFinishedCounter.Inc(); AFL_VERIFY(CompactionsLimit.Dec() >= 0); @@ -52,9 +48,6 @@ bool TReadOnlyController::DoOnWriteIndexStart(const ui64 tabletId, NOlap::TColum if (change.TypeString() == NOlap::TTTLColumnEngineChanges::StaticTypeName()) { TTLStartedCounter.Inc(); } - if (change.TypeString() == NOlap::TInsertColumnEngineChanges::StaticTypeName()) { - InsertStartedCounter.Inc(); - } if (change.TypeString() == NOlap::TCompactColumnEngineChanges::StaticTypeName()) { CompactionStartedCounter.Inc(); } diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index a13776983451..851e6566b084 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -13,8 +13,6 @@ class TReadOnlyController: public ICSController { private: YDB_READONLY(TAtomicCounter, TTLFinishedCounter, 0); YDB_READONLY(TAtomicCounter, TTLStartedCounter, 0); - YDB_READONLY(TAtomicCounter, InsertFinishedCounter, 0); - YDB_READONLY(TAtomicCounter, InsertStartedCounter, 0); YDB_READONLY(TAtomicCounter, CompactionFinishedCounter, 0); YDB_READONLY(TAtomicCounter, CompactionStartedCounter, 0); YDB_READONLY(TAtomicCounter, CleaningFinishedCounter, 0); @@ -99,19 +97,6 @@ class TReadOnlyController: public ICSController { return count > 0; } - void WaitIndexation(const TDuration d) const { - TInstant start = TInstant::Now(); - ui32 insertsStart = GetInsertStartedCounter().Val(); - while (Now() - start < d) { - if (insertsStart != GetInsertStartedCounter().Val()) { - insertsStart = GetInsertStartedCounter().Val(); - start = TInstant::Now(); - } - Cerr << "WAIT_INDEXATION: " << GetInsertStartedCounter().Val() << Endl; - Sleep(TDuration::Seconds(1)); - } - } - bool WaitCleaning(const TDuration d, NActors::TTestBasicRuntime* testRuntime = nullptr) const { TInstant start = TInstant::Now(); const ui32 countStart0 = GetCleaningStartedCounter().Val(); diff --git a/ydb/core/tx/columnshard/inflight_request_tracker.cpp b/ydb/core/tx/columnshard/inflight_request_tracker.cpp index 52c6b17e0d83..89ed6b17b047 100644 --- a/ydb/core/tx/columnshard/inflight_request_tracker.cpp +++ b/ydb/core/tx/columnshard/inflight_request_tracker.cpp @@ -1,10 +1,11 @@ #include "columnshard_impl.h" #include "columnshard_schema.h" #include "inflight_request_tracker.h" -#include "tablet/ext_tx_base.h" + #include "engines/column_engine.h" #include "engines/reader/plain_reader/constructor/read_metadata.h" #include "hooks/abstract/abstract.h" +#include "tablet/ext_tx_base.h" namespace NKikimr::NColumnShard { @@ -16,20 +17,9 @@ NOlap::NReader::TReadMetadataBase::TConstPtr TInFlightReadsTracker::ExtractInFli const NOlap::NReader::TReadMetadataBase::TConstPtr readMetaBase = it->second; { - { - auto it = SnapshotsLive.find(readMetaBase->GetRequestSnapshot()); - AFL_VERIFY(it != SnapshotsLive.end()); - Y_UNUSED(it->second.DelRequest(cookie, now)); - } - - if (NOlap::NReader::NPlain::TReadMetadata::TConstPtr readMeta = - std::dynamic_pointer_cast(readMetaBase)) { - auto insertStorage = StoragesManager->GetInsertOperator(); - auto tracker = insertStorage->GetBlobsTracker(); - for (const auto& committedBlob : readMeta->CommittedBlobs) { - tracker->FreeBlob(committedBlob.GetBlobRange().GetBlobId()); - } - } + auto it = SnapshotsLive.find(readMetaBase->GetRequestSnapshot()); + AFL_VERIFY(it != SnapshotsLive.end()); + Y_UNUSED(it->second.DelRequest(cookie, now)); } Counters->OnSnapshotsInfo(SnapshotsLive.size(), GetSnapshotToClean()); @@ -50,12 +40,6 @@ void TInFlightReadsTracker::AddToInFlightRequest( auto selectInfo = readMeta->SelectInfo; Y_ABORT_UNLESS(selectInfo); SelectStatsDelta += selectInfo->Stats(); - - auto insertStorage = StoragesManager->GetInsertOperator(); - auto tracker = insertStorage->GetBlobsTracker(); - for (const auto& committedBlob : readMeta->CommittedBlobs) { - tracker->UseBlob(committedBlob.GetBlobRange().GetBlobId()); - } } namespace { @@ -139,8 +123,7 @@ bool TInFlightReadsTracker::LoadFromDatabase(NTable::TDatabase& tableDB) { return true; } -ui64 TInFlightReadsTracker::AddInFlightRequest( - NOlap::NReader::TReadMetadataBase::TConstPtr readMeta, const NOlap::TVersionedIndex* index) { +ui64 TInFlightReadsTracker::AddInFlightRequest(NOlap::NReader::TReadMetadataBase::TConstPtr readMeta, const NOlap::TVersionedIndex* index) { const ui64 cookie = NextCookie++; auto it = SnapshotsLive.find(readMeta->GetRequestSnapshot()); if (it == SnapshotsLive.end()) { diff --git a/ydb/core/tx/columnshard/loading/stages.cpp b/ydb/core/tx/columnshard/loading/stages.cpp index 855b987dd826..64952e9b926f 100644 --- a/ydb/core/tx/columnshard/loading/stages.cpp +++ b/ydb/core/tx/columnshard/loading/stages.cpp @@ -8,27 +8,6 @@ namespace NKikimr::NColumnShard::NLoading { -bool TInsertTableInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { - NIceDb::TNiceDb db(txc.DB); - TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - auto localInsertTable = std::make_unique(); - for (auto&& i : Self->TablesManager.GetTables()) { - localInsertTable->RegisterPathInfo(i.first); - } - if (!localInsertTable->Load(db, dbTable, TAppData::TimeProvider->Now())) { - ACFL_ERROR("step", "TInsertTable::Load_Fails"); - return false; - } - Self->InsertTable.swap(localInsertTable); - return true; -} - -bool TInsertTableInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { - NIceDb::TNiceDb db(txc.DB); - return Schema::Precharge(db, txc.DB.GetScheme()); -} - bool TTxControllerInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { auto localTxController = std::make_unique(*Self); if (!localTxController->Load(txc)) { @@ -72,40 +51,6 @@ bool TStoragesManagerInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionC (int)Schema::Precharge(db, txc.DB.GetScheme()); } -bool TLongTxInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { - NIceDb::TNiceDb db(txc.DB); - auto rowset = db.Table().Select(); - if (!rowset.IsReady()) { - return false; - } - - while (!rowset.EndOfSet()) { - const TInsertWriteId writeId = (TInsertWriteId)rowset.GetValue(); - const ui32 writePartId = rowset.GetValue(); - NKikimrLongTxService::TLongTxId proto; - Y_ABORT_UNLESS(proto.ParseFromString(rowset.GetValue())); - const auto longTxId = NLongTxService::TLongTxId::FromProto(proto); - - std::optional granuleShardingVersion; - if (rowset.HaveValue() && - rowset.GetValue()) { - granuleShardingVersion = rowset.GetValue(); - } - - Self->LoadLongTxWrite(writeId, writePartId, longTxId, granuleShardingVersion); - - if (!rowset.Next()) { - return false; - } - } - return true; -} - -bool TLongTxInitializer::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { - NIceDb::TNiceDb db(txc.DB); - return Schema::Precharge(db, txc.DB.GetScheme()); -} - bool TDBLocksInitializer::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { if (txc.DB.GetScheme().GetTableInfo(Schema::Locks::TableId)) { TColumnShardLocksDb locksDb(*Self, txc); diff --git a/ydb/core/tx/columnshard/loading/stages.h b/ydb/core/tx/columnshard/loading/stages.h index 7cc2bb783bce..f2c6493e169e 100644 --- a/ydb/core/tx/columnshard/loading/stages.h +++ b/ydb/core/tx/columnshard/loading/stages.h @@ -22,16 +22,6 @@ class ITxShardInitReader: public ITxReader { } }; -class TInsertTableInitializer: public ITxShardInitReader { -private: - using TBase = ITxShardInitReader; - virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; - virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; - -public: - using TBase::TBase; -}; - class TTxControllerInitializer: public ITxShardInitReader { private: using TBase = ITxShardInitReader; @@ -58,15 +48,6 @@ class TStoragesManagerInitializer: public ITxShardInitReader { virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; -public: - using TBase::TBase; -}; -class TLongTxInitializer: public ITxShardInitReader { -private: - using TBase = ITxShardInitReader; - virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; - virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) override; - public: using TBase::TBase; }; diff --git a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h index f852dd5c68ee..b1e4f574feb7 100644 --- a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h +++ b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h @@ -60,7 +60,7 @@ enum class ENormalizerSequentialId : ui32 { DeprecatedPortionsMetadata, CleanGranuleId, DeprecatedEmptyPortionsCleaner, - CleanInsertionDedup, + DeprecatedCleanInsertionDedup, GCCountersNormalizer, DeprecatedRestorePortionFromChunks, SyncPortionFromChunks, diff --git a/ydb/core/tx/columnshard/normalizer/ya.make b/ydb/core/tx/columnshard/normalizer/ya.make index f48c3308ac38..00a235646bbc 100644 --- a/ydb/core/tx/columnshard/normalizer/ya.make +++ b/ydb/core/tx/columnshard/normalizer/ya.make @@ -6,7 +6,6 @@ PEERDIR( ydb/core/tx/columnshard/normalizer/tablet ydb/core/tx/columnshard/normalizer/tables ydb/core/tx/columnshard/normalizer/portion - ydb/core/tx/columnshard/normalizer/insert_table ydb/core/tx/columnshard/normalizer/schema_version ) diff --git a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp index f501fb17535e..ddf8628b80c8 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp @@ -53,7 +53,7 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { const std::vector> defaultFields = Context.GetActualSchema()->GetAbsentFields(batch.GetContainer()->schema()); if (defaultFields.empty()) { - if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { + if (!Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batch.GetContainer(), Context); NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); @@ -85,7 +85,7 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { } case NEvWrite::EModificationType::Replace: case NEvWrite::EModificationType::Delete: { - if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { + if (!Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batch.GetContainer(), Context); NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); diff --git a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp index 41945707eaf2..94d800b2f71c 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/restore.cpp @@ -49,7 +49,7 @@ NKikimr::TConclusionStatus TModificationRestoreTask::DoOnFinished() { } auto batchResult = Merger->BuildResultBatch(); - if (!WriteData.GetWritePortions() || !Context.GetNoTxWrite()) { + if (!Context.GetNoTxWrite()) { std::shared_ptr task = std::make_shared(std::move(WriteData), batchResult.GetContainer(), Context); NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); diff --git a/ydb/core/tx/columnshard/operations/manager.cpp b/ydb/core/tx/columnshard/operations/manager.cpp index ef7d66aaaa92..6b0d6adc0c76 100644 --- a/ydb/core/tx/columnshard/operations/manager.cpp +++ b/ydb/core/tx/columnshard/operations/manager.cpp @@ -29,7 +29,7 @@ bool TOperationsManager::Load(NTabletFlatExecutor::TTransactionContext& txc) { Y_ABORT_UNLESS(metaProto.ParseFromString(metadata)); auto operation = std::make_shared(TUnifiedPathId{}, writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), - granuleShardingVersionId, NEvWrite::EModificationType::Upsert, false); + granuleShardingVersionId, NEvWrite::EModificationType::Upsert); operation->FromProto(metaProto); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_TX)("event", "register_operation_on_load")("operation_id", operation->GetWriteId()); AFL_VERIFY(operation->GetStatus() != EOperationStatus::Draft); @@ -203,10 +203,10 @@ void TOperationsManager::LinkTransactionOnComplete(const ui64 /*lockId*/, const } TWriteOperation::TPtr TOperationsManager::RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, - const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool portionsWriting) { + const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType) { auto writeId = BuildNextOperationWriteId(); auto operation = std::make_shared(pathId, writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), - granuleShardingVersionId, mType, portionsWriting); + granuleShardingVersionId, mType); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "register_operation")("operation_id", operation->GetWriteId())( "last", LastWriteId); AFL_VERIFY(Operations.emplace(operation->GetWriteId(), operation).second); diff --git a/ydb/core/tx/columnshard/operations/manager.h b/ydb/core/tx/columnshard/operations/manager.h index 3c381498c7c8..4937974af1c5 100644 --- a/ydb/core/tx/columnshard/operations/manager.h +++ b/ydb/core/tx/columnshard/operations/manager.h @@ -200,7 +200,7 @@ class TOperationsManager { } TWriteOperation::TPtr RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, - const NEvWrite::EModificationType mType, const bool portionsWriting); + const NEvWrite::EModificationType mType); bool RegisterLock(const ui64 lockId, const ui64 generationId) { if (LockFeatures.contains(lockId)) { return false; diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp index 34429f20b351..f7b4502d5b2c 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp @@ -1,8 +1,8 @@ -#include #include "builder.h" #include #include +#include #include #include #include @@ -76,21 +76,20 @@ class TPortionWriteController: public NColumnShard::IWriteController, portions.emplace_back(i.ExtractPortion()); } NColumnShard::TInsertedPortions pack({ WriteResult }, std::move(portions)); - auto result = std::make_unique( - putResult->GetPutStatus(), Action, std::move(pack)); + auto result = + std::make_unique(putResult->GetPutStatus(), Action, std::move(pack)); ctx.Send(DstActor, result.release()); } virtual void DoOnStartSending() override { } public: - TPortionWriteController(const TActorId& dstActor, const std::shared_ptr& action, const NColumnShard::TWriteResult& writeResult, - std::vector&& portions) + TPortionWriteController(const TActorId& dstActor, const std::shared_ptr& action, + const NColumnShard::TWriteResult& writeResult, std::vector&& portions) : Action(action) , Portions(std::move(portions)) , WriteResult(writeResult) - , DstActor(dstActor) - { + , DstActor(dstActor) { for (auto&& p : Portions) { for (auto&& b : p.MutablePortion().MutableBlobs()) { auto& task = AddWriteTask(TBlobWriteInfo::BuildWriteTask(b.GetResultBlob(), action)); @@ -101,9 +100,9 @@ class TPortionWriteController: public NColumnShard::IWriteController, }; void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { - const NActors::TLogContextGuard g = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletId)( - "parent_id", Context.GetTabletActorId())("write_id", WriteData.GetWriteMeta().GetWriteId())( - "path_id", WriteData.GetWriteMeta().GetPathId()); + const NActors::TLogContextGuard g = + NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletId)("parent_id", + Context.GetTabletActorId())("write_id", WriteData.GetWriteMeta().GetWriteId())("path_id", WriteData.GetWriteMeta().GetPathId()); if (!Context.IsActive()) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "abort_execution"); ReplyError("execution aborted", NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Internal); @@ -114,80 +113,38 @@ void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { ReplyError("no data in batch", NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Internal); return; } - if (WriteData.GetWritePortions()) { - if (OriginalBatch->num_rows() == 0) { - NColumnShard::TWriteResult wResult(WriteData.GetWriteMetaPtr(), WriteData.GetSize(), nullptr, true, 0); - NColumnShard::TInsertedPortions pack({ wResult }, {}); - auto result = std::make_unique( - NKikimrProto::EReplyStatus::OK, nullptr, std::move(pack)); - NActors::TActivationContext::AsActorContext().Send(Context.GetTabletActorId(), result.release()); - } else { - std::shared_ptr pkBatch = - NArrow::TColumnOperator().Extract(OriginalBatch, Context.GetActualSchema()->GetIndexInfo().GetPrimaryKey()->fields()); - auto batches = NArrow::NMerger::TRWSortableBatchPosition::SplitByBordersInIntervalPositions(OriginalBatch, - Context.GetActualSchema()->GetIndexInfo().GetPrimaryKey()->field_names(), WriteData.GetData()->GetSeparationPoints()); - NColumnShard::TWriteResult wResult(WriteData.GetWriteMetaPtr(), WriteData.GetSize(), pkBatch, false, OriginalBatch->num_rows()); - std::vector portions; - for (auto&& batch : batches) { - if (!batch) { - continue; - } - auto portionConclusion = - Context.GetActualSchema()->PrepareForWrite(Context.GetActualSchema(), WriteData.GetWriteMeta().GetPathId().InternalPathId, batch, - WriteData.GetWriteMeta().GetModificationType(), Context.GetStoragesManager(), Context.GetSplitterCounters()); - if (portionConclusion.IsFail()) { - ReplyError(portionConclusion.GetErrorMessage(), NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Request); - return; - } - portions.emplace_back(portionConclusion.DetachResult()); - } - auto writeController = std::make_shared( - Context.GetTabletActorId(), WriteData.GetBlobsAction(), wResult, std::move(portions)); - if (WriteData.GetBlobsAction()->NeedDraftTransaction()) { - TActorContext::AsActorContext().Send( - Context.GetTabletActorId(), std::make_unique(writeController)); - } else { - TActorContext::AsActorContext().Register(NColumnShard::CreateWriteActor(TabletId, writeController, TInstant::Max())); - } - } + if (OriginalBatch->num_rows() == 0) { + NColumnShard::TWriteResult wResult(WriteData.GetWriteMetaPtr(), WriteData.GetSize(), nullptr, true, 0); + NColumnShard::TInsertedPortions pack({ wResult }, {}); + auto result = std::make_unique( + NKikimrProto::EReplyStatus::OK, nullptr, std::move(pack)); + NActors::TActivationContext::AsActorContext().Send(Context.GetTabletActorId(), result.release()); } else { - const auto& indexSchema = Context.GetActualSchema()->GetIndexInfo().ArrowSchema(); - auto subsetConclusion = NArrow::TColumnOperator().IgnoreOnDifferentFieldTypes().BuildSequentialSubset(OriginalBatch, indexSchema); - if (subsetConclusion.IsFail()) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "unadaptable schemas")("index", indexSchema.ToString())( - "problem", subsetConclusion.GetErrorMessage()); - ReplyError("unadaptable schema: " + subsetConclusion.GetErrorMessage(), - NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Internal); - return; - } - NArrow::TSchemaSubset subset = subsetConclusion.DetachResult(); - - if (OriginalBatch->num_columns() != indexSchema.num_fields()) { - AFL_VERIFY(OriginalBatch->num_columns() < indexSchema.num_fields())("original", OriginalBatch->num_columns())( - "index", indexSchema.num_fields()); - if (HasAppData() && !AppDataVerified().FeatureFlags.GetEnableOptionalColumnsInColumnShard() && - WriteData.GetWriteMeta().GetModificationType() != NEvWrite::EModificationType::Delete) { - subset = NArrow::TSchemaSubset::AllFieldsAccepted(); - const auto columnIdsVector = Context.GetActualSchema()->GetIndexInfo().GetColumnIds(false); - const std::set columnIdsSet(columnIdsVector.begin(), columnIdsVector.end()); - auto normalized = - Context.GetActualSchema() - ->NormalizeBatch(*Context.GetActualSchema(), std::make_shared(OriginalBatch), columnIdsSet) - .DetachResult(); - OriginalBatch = NArrow::ToBatch(normalized->BuildTableVerified()); + std::shared_ptr pkBatch = + NArrow::TColumnOperator().Extract(OriginalBatch, Context.GetActualSchema()->GetIndexInfo().GetPrimaryKey()->fields()); + auto batches = NArrow::NMerger::TRWSortableBatchPosition::SplitByBordersInIntervalPositions( + OriginalBatch, Context.GetActualSchema()->GetIndexInfo().GetPrimaryKey()->field_names(), WriteData.GetData()->GetSeparationPoints()); + NColumnShard::TWriteResult wResult(WriteData.GetWriteMetaPtr(), WriteData.GetSize(), pkBatch, false, OriginalBatch->num_rows()); + std::vector portions; + for (auto&& batch : batches) { + if (!batch) { + continue; + } + auto portionConclusion = Context.GetActualSchema()->PrepareForWrite(Context.GetActualSchema(), WriteData.GetWriteMeta().GetPathId().InternalPathId, + batch, WriteData.GetWriteMeta().GetModificationType(), Context.GetStoragesManager(), Context.GetSplitterCounters()); + if (portionConclusion.IsFail()) { + ReplyError(portionConclusion.GetErrorMessage(), NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Request); + return; } + portions.emplace_back(portionConclusion.DetachResult()); } - auto batches = BuildSlices(); - if (batches) { - auto writeDataPtr = std::make_shared(std::move(WriteData)); - writeDataPtr->SetSchemaSubset(std::move(subset)); - std::shared_ptr pkBatch = - NArrow::TColumnOperator().Extract(OriginalBatch, Context.GetActualSchema()->GetIndexInfo().GetPrimaryKey()->fields()); - auto result = std::make_unique(writeDataPtr, std::move(*batches), pkBatch); - TActorContext::AsActorContext().Send(Context.GetBufferizationInsertionActorId(), result.release()); + auto writeController = std::make_shared( + Context.GetTabletActorId(), WriteData.GetBlobsAction(), wResult, std::move(portions)); + if (WriteData.GetBlobsAction()->NeedDraftTransaction()) { + TActorContext::AsActorContext().Send( + Context.GetTabletActorId(), std::make_unique(writeController)); } else { - ReplyError("Cannot slice input to batches", NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass::Internal); - return; + TActorContext::AsActorContext().Register(NColumnShard::CreateWriteActor(TabletId, writeController, TInstant::Max())); } } } diff --git a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h index cbcc954630db..a511f92395ff 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h +++ b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h @@ -20,7 +20,6 @@ class TWriteUnit: public NColumnShard::TMonitoringObjectsCounter { : Data(data) , Batch(batch) { Data->MutableWriteMeta().OnStage(NEvWrite::EWriteStage::WaitFlush); - AFL_VERIFY(Data->GetWritePortions()); AFL_VERIFY(Batch.HasContainer()); } }; diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 11e6c9d4ff92..6318a303830f 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -16,7 +16,7 @@ namespace NKikimr::NColumnShard { TWriteOperation::TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional granuleShardingVersionId, - const NEvWrite::EModificationType mType, const bool writePortions) + const NEvWrite::EModificationType mType) : PathId(pathId) , Status(status) , CreatedAt(createdAt) @@ -24,8 +24,7 @@ TWriteOperation::TWriteOperation(const TUnifiedPathId& pathId, const TOperationW , LockId(lockId) , Cookie(cookie) , GranuleShardingVersionId(granuleShardingVersionId) - , ModificationType(mType) - , WritePortions(writePortions) { + , ModificationType(mType) { } void TWriteOperation::Start( @@ -38,7 +37,7 @@ void TWriteOperation::Start( writeMeta->SetLockId(LockId); writeMeta->SetModificationType(ModificationType); NEvWrite::TWriteData writeData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), - owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR), WritePortions); + owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR)); std::shared_ptr task = std::make_shared(std::move(writeData), context); NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); @@ -52,31 +51,16 @@ void TWriteOperation::CommitOnExecute( TBlobGroupSelector dsGroupSelector(owner.Info()); NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - if (!WritePortions) { - THashSet insertWriteIds(InsertWriteIds.begin(), InsertWriteIds.end()); - auto pathExists = [&](TInternalPathId pathId) { - return owner.TablesManager.HasTable(pathId); - }; - if (insertWriteIds.size()) { - const auto counters = owner.InsertTable->Commit(dbTable, snapshot.GetPlanStep(), snapshot.GetTxId(), insertWriteIds, pathExists); - owner.Counters.GetTabletCounters()->OnWriteCommitted(counters); - } - } else { - for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnExecute(txc, i, snapshot); - } + for (auto&& i : InsertWriteIds) { + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnExecute(txc, i, snapshot); } } void TWriteOperation::CommitOnComplete(TColumnShard& owner, const NOlap::TSnapshot& /*snapshot*/) const { Y_ABORT_UNLESS(Status == EOperationStatus::Prepared || InsertWriteIds.empty()); - if (!WritePortions) { - owner.UpdateInsertTableCounters(); - } else { - for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnComplete( - i, owner.MutableIndexAs()); - } + for (auto&& i : InsertWriteIds) { + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).CommitPortionOnComplete( + i, owner.MutableIndexAs()); } } @@ -110,7 +94,7 @@ void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& prot proto.AddInternalWriteIds((ui64)writeId); } proto.SetModificationType((ui32)ModificationType); - proto.SetWritePortions(WritePortions); + proto.SetWritePortions(true); PathId.InternalPathId.ToProto(proto); } @@ -118,9 +102,8 @@ void TWriteOperation::FromProto(const NKikimrTxColumnShard::TInternalOperationDa for (auto&& writeId : proto.GetInternalWriteIds()) { InsertWriteIds.push_back(TInsertWriteId(writeId)); } - WritePortions = proto.GetWritePortions(); PathId.InternalPathId = TInternalPathId::FromProto(proto); - AFL_VERIFY(!WritePortions || PathId.InternalPathId); + AFL_VERIFY(PathId.InternalPathId); if (proto.HasModificationType()) { ModificationType = (NEvWrite::EModificationType)proto.GetModificationType(); } else { @@ -134,25 +117,17 @@ void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::T TBlobGroupSelector dsGroupSelector(owner.Info()); NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - if (!WritePortions) { - THashSet writeIds; - writeIds.insert(InsertWriteIds.begin(), InsertWriteIds.end()); - owner.InsertTable->Abort(dbTable, writeIds); - } else { - for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnExecute( - txc, i, owner.GetCurrentSnapshotForInternalModification()); - } + for (auto&& i : InsertWriteIds) { + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnExecute( + txc, i, owner.GetCurrentSnapshotForInternalModification()); } } void TWriteOperation::AbortOnComplete(TColumnShard& owner) const { Y_ABORT_UNLESS(Status != EOperationStatus::Draft); - if (WritePortions) { - for (auto&& i : InsertWriteIds) { - owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnComplete( - i, owner.MutableIndexAs()); - } + for (auto&& i : InsertWriteIds) { + owner.MutableIndexAs().MutableGranuleVerified(PathId.InternalPathId).AbortPortionOnComplete( + i, owner.MutableIndexAs()); } } diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index d9276d26785d..0eb1cf172c67 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -60,7 +60,6 @@ class TWriteOperation: public TMonitoringObjectsCounter { YDB_ACCESSOR(EOperationBehaviour, Behaviour, EOperationBehaviour::Undefined); YDB_READONLY_DEF(std::optional, GranuleShardingVersionId); YDB_READONLY(NEvWrite::EModificationType, ModificationType, NEvWrite::EModificationType::Upsert); - bool WritePortions = false; const std::shared_ptr Activity = std::make_shared(1); public: @@ -71,8 +70,7 @@ class TWriteOperation: public TMonitoringObjectsCounter { } TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, - const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, - const bool writePortions); + const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType); void Start( TColumnShard& owner, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const NOlap::TWritingContext& context); diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 845949cb62b8..0207b8670076 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -9,8 +9,7 @@ namespace NKikimr::NColumnShard { bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { owner->Counters.GetCSCounters().WritingCounters->OnWritingTaskDequeue(TMonotonic::Now() - Created); owner->OperationsManager->RegisterLock(LockId, owner->Generation()); - auto writeOperation = owner->OperationsManager->RegisterOperation(PathId, - LockId, Cookie, GranuleShardingVersionId, ModificationType, AppDataVerified().FeatureFlags.GetEnableWritePortionsOnInsert()); + auto writeOperation = owner->OperationsManager->RegisterOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", ArrowData->GetSize())("operation_id", writeOperation->GetIdentifier())( "in_flight", owner->Counters.GetWritesMonitor()->GetWritesInFlight())( diff --git a/ydb/core/tx/columnshard/test_helper/controllers.cpp b/ydb/core/tx/columnshard/test_helper/controllers.cpp index a9f1a877a13b..f169e681ab4f 100644 --- a/ydb/core/tx/columnshard/test_helper/controllers.cpp +++ b/ydb/core/tx/columnshard/test_helper/controllers.cpp @@ -1,7 +1,6 @@ #include "columnshard_ut_common.h" #include "controllers.h" -#include #include namespace NKikimr::NOlap { diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp deleted file mode 100644 index 99298434b88b..000000000000 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "long_tx_write.h" -#include - -namespace NKikimr::NColumnShard { - -TLongTxTransactionOperator::TProposeResult TLongTxTransactionOperator::DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& /*txc*/) { - if (WriteIds.empty()) { - return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, - TStringBuilder() << "Commit TxId# " << GetTxId() << " has an empty list of write ids"); - } - - for (auto&& writeId : WriteIds) { - if (!owner.LongTxWrites.contains(writeId)) { - return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, - TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " that no longer exists"); - } - auto& lw = owner.LongTxWrites[writeId]; - if (lw.PreparedTxId != 0) { - return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, - TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " that is already locked by TxId# " << lw.PreparedTxId); - } - - if (auto* inserted = owner.InsertTable->GetInserted().GetOptional(writeId)) { - auto granuleShardingInfo = - owner.GetIndexAs().GetVersionedIndex().GetShardingInfoActual(inserted->GetPathId()); - if (granuleShardingInfo && lw.GranuleShardingVersionId && *lw.GranuleShardingVersionId != granuleShardingInfo->GetSnapshotVersion()) { - return TProposeResult(NKikimrTxColumnShard::EResultStatus::ERROR, - TStringBuilder() << "Commit TxId# " << GetTxId() << " references WriteId# " << (ui64)writeId << " declined through sharding deprecated"); - } - } - } - - for (auto&& writeId : WriteIds) { - owner.AddLongTxWrite(writeId, GetTxId()); - } - return TProposeResult(); -} - -bool TLongTxTransactionOperator::DoParse(TColumnShard& /*owner*/, const TString& data) { - NKikimrTxColumnShard::TCommitTxBody commitTxBody; - if (!commitTxBody.ParseFromString(data)) { - return false; - } - - for (auto& id : commitTxBody.GetWriteIds()) { - WriteIds.insert(TInsertWriteId{ id }); - } - return true; -} - -void TLongTxTransactionOperator::DoSendReply(TColumnShard& owner, const TActorContext& ctx) { - const auto& txInfo = GetTxInfo(); - ctx.Send(txInfo.Source, BuildProposeResultEvent(owner).release()); -} - -} diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h deleted file mode 100644 index 00b588c037ab..000000000000 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h +++ /dev/null @@ -1,104 +0,0 @@ -#pragma once - -#include "propose_tx.h" - -#include -#include - -namespace NKikimr::NColumnShard { - - class TLongTxTransactionOperator: public IProposeTxOperator, public TMonitoringObjectsCounter { - using TBase = IProposeTxOperator; - using TProposeResult = TTxController::TProposeResult; - static inline auto Registrator = TFactory::TRegistrator(NKikimrTxColumnShard::TX_KIND_COMMIT); - - private: - virtual TString DoDebugString() const override { - return "LONG_TX_WRITE"; - } - - bool TxWithDeadline() const override { - return true; - } - - virtual TProposeResult DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override; - virtual void DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override { - - } - virtual void DoSendReply(TColumnShard& owner, const TActorContext& ctx) override; - - virtual void DoFinishProposeOnExecute(TColumnShard& /*owner*/, NTabletFlatExecutor::TTransactionContext& /*txc*/) override { - } - virtual void DoFinishProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override { - } - virtual TString DoGetOpType() const override { - return "LongTxWrite"; - } - virtual bool DoIsAsync() const override { - return false; - } - virtual bool DoParse(TColumnShard& owner, const TString& data) override; - virtual bool DoCheckTxInfoForReply(const TFullTxInfo& /*originalTxInfo*/) const override { - return true; - } - virtual bool DoCheckAllowUpdate(const TFullTxInfo& currentTxInfo) const override { - return (currentTxInfo.Source == GetTxInfo().Source && currentTxInfo.Cookie == GetTxInfo().Cookie); - } - public: - using TBase::TBase; - - virtual void DoOnTabletInit(TColumnShard& owner) override { - for (auto&& writeId : WriteIds) { - AFL_VERIFY(owner.LongTxWrites.contains(writeId))("problem", "ltx_not_exists_for_write_id")("txId", GetTxId())("writeId", (ui64)writeId); - owner.AddLongTxWrite(writeId, GetTxId()); - } - } - - bool ProgressOnExecute(TColumnShard& owner, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) override { - TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - - auto pathExists = [&](TInternalPathId pathId) { - return owner.TablesManager.HasTable(pathId); - }; - - auto counters = owner.InsertTable->Commit(dbTable, version.GetPlanStep(), version.GetTxId(), WriteIds, pathExists); - - owner.Counters.GetTabletCounters()->IncCounter(COUNTER_BLOBS_COMMITTED, counters.Rows); - owner.Counters.GetTabletCounters()->IncCounter(COUNTER_BYTES_COMMITTED, counters.Bytes); - owner.Counters.GetTabletCounters()->IncCounter(COUNTER_RAW_BYTES_COMMITTED, counters.RawBytes); - - NIceDb::TNiceDb db(txc.DB); - for (TInsertWriteId writeId : WriteIds) { - AFL_VERIFY(owner.RemoveLongTxWrite(db, writeId, GetTxId())); - } - owner.UpdateInsertTableCounters(); - return true; - } - - bool ProgressOnComplete(TColumnShard& owner, const TActorContext& ctx) override { - auto result = std::make_unique(owner.TabletID(), TxInfo.TxKind, GetTxId(), NKikimrTxColumnShard::SUCCESS); - result->Record.SetStep(TxInfo.PlanStep); - ctx.Send(TxInfo.Source, result.release(), 0, TxInfo.Cookie); - return true; - } - - virtual bool ExecuteOnAbort(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) override { - NIceDb::TNiceDb db(txc.DB); - for (TInsertWriteId writeId : WriteIds) { - AFL_VERIFY(owner.RemoveLongTxWrite(db, writeId, GetTxId())); - } - TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - owner.InsertTable->Abort(dbTable, WriteIds); - return true; - } - virtual bool CompleteOnAbort(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) override { - return true; - } - - private: - THashSet WriteIds; - }; - -} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/transactions/operators/ya.make b/ydb/core/tx/columnshard/transactions/operators/ya.make index e96323b51b44..238fb7a1a36f 100644 --- a/ydb/core/tx/columnshard/transactions/operators/ya.make +++ b/ydb/core/tx/columnshard/transactions/operators/ya.make @@ -2,7 +2,6 @@ LIBRARY() SRCS( GLOBAL schema.cpp - GLOBAL long_tx_write.cpp GLOBAL backup.cpp GLOBAL sharing.cpp propose_tx.cpp diff --git a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp index 29595cec6170..f2b8b7ed26b5 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_columnshard_read_write.cpp @@ -10,10 +10,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -436,10 +436,9 @@ void TestWrite(const TestTableDescription& table) { UNIT_ASSERT(ok); } -void TestWriteOverload(const TestTableDescription& table, bool WritePortionsOnInsert) { +void TestWriteOverload(const TestTableDescription& table) { TTestBasicRuntime runtime; TTester::Setup(runtime); - runtime.GetAppData().FeatureFlags.SetEnableWritePortionsOnInsert(WritePortionsOnInsert); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csDefaultControllerGuard->SetOverrideBlobSplitSettings(std::nullopt); TActorId sender = runtime.AllocateEdgeActor(); @@ -467,19 +466,10 @@ void TestWriteOverload(const TestTableDescription& table, bool WritePortionsOnIn auto captureEvents = [&](TTestActorRuntimeBase&, TAutoPtr& ev) { if (toCatch) { TAutoPtr eventToCapture; - if (WritePortionsOnInsert) { - if (auto* msg = TryGetPrivateEvent(ev)) { - Cerr << "CATCH TEvWritePortionResult, status " << msg->GetWriteStatus() << Endl; - if (msg->GetWriteStatus() != NKikimrProto::EReplyStatus::UNKNOWN) { - eventToCapture = ev.Release(); - } - } - } else { - if (auto* msg = TryGetPrivateEvent(ev)) { - Cerr << "CATCH TEvWrite, status " << msg->GetPutResult().GetPutStatus() << Endl; - if (msg->GetPutResult().GetPutStatus() != NKikimrProto::UNKNOWN) { - eventToCapture = ev.Release(); - } + if (auto* msg = TryGetPrivateEvent(ev)) { + Cerr << "CATCH TEvWritePortionResult, status " << msg->GetWriteStatus() << Endl; + if (msg->GetWriteStatus() != NKikimrProto::EReplyStatus::UNKNOWN) { + eventToCapture = ev.Release(); } } if (eventToCapture) { @@ -1398,7 +1388,7 @@ void TestReadWithProgramNoProjection(const TestTableDescription& table = {}) { //remove projections auto* commands = ssa.MutableCommand(); - for(int i = commands->size() - 1; i >= 0; --i) { + for (int i = commands->size() - 1; i >= 0; --i) { if ((*commands)[i].HasProjection()) { commands->DeleteSubrange(i, 1); } @@ -1415,19 +1405,19 @@ void TestReadWithProgramNoProjection(const TestTableDescription& table = {}) { TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep, txId)); reader.SetProgram(programText); auto rb = reader.ReadAll(); - switch(i) { + switch (i) { case 0: - UNIT_ASSERT(reader.IsError()); - break; + UNIT_ASSERT(reader.IsError()); + break; case 1: - UNIT_ASSERT(!reader.IsError()); - break; + UNIT_ASSERT(!reader.IsError()); + break; case 2: - UNIT_ASSERT(reader.IsError()); - UNIT_ASSERT(reader.GetErrors().back().Getmessage().Contains("program has no projections")); - break; + UNIT_ASSERT(reader.IsError()); + UNIT_ASSERT(reader.GetErrors().back().Getmessage().Contains("program has no projections")); + break; } UNIT_ASSERT(reader.IsFinished()); ++i; @@ -1743,10 +1733,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { TestWrite(table); } - Y_UNIT_TEST_QUATRO(WriteOverload, InStore, WithWritePortionsOnInsert) { + Y_UNIT_TEST_DUO(WriteOverload, InStore) { TestTableDescription table; table.InStore = InStore; - TestWriteOverload(table, WithWritePortionsOnInsert); + TestWriteOverload(table); } Y_UNIT_TEST(WriteReadDuplicate) { @@ -2485,7 +2475,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { // Cerr << response << Endl; UNIT_ASSERT_VALUES_EQUAL(response.GetStatus(), Ydb::StatusIds::BAD_REQUEST); UNIT_ASSERT_VALUES_EQUAL(response.IssuesSize(), 1); - UNIT_ASSERT_STRING_CONTAINS(response.GetIssues(0).message(), TStringBuilder() << "Snapshot too old: {" << planStep - staleness.MilliSeconds() << ":max}"); + UNIT_ASSERT_STRING_CONTAINS( + response.GetIssues(0).message(), TStringBuilder() << "Snapshot too old: {" << planStep - staleness.MilliSeconds() << ":max}"); } // Try to read snapshot that is too old @@ -2497,11 +2488,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } } - template + template void TestCompactionGC() { TTestBasicRuntime runtime; auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); - csDefaultControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); csDefaultControllerGuard->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); csDefaultControllerGuard->SetOverrideBlobSplitSettings(NOlap::NSplitter::TSplitSettings()); TTester::Setup(runtime); @@ -2662,7 +2652,6 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { UNIT_ASSERT(reader.IsCorrectlyFinished()); UNIT_ASSERT(CheckOrdered(rb)); UNIT_ASSERT(reader.GetIterationsCount() < 10); - csDefaultControllerGuard->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); // We captured EvReadFinished event and dropped is so the columnshard still thinks that // read request is in progress and keeps the portions @@ -2682,7 +2671,6 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } Cerr << "Compactions happened: " << csDefaultControllerGuard->GetCompactionStartedCounter().Val() << Endl; - Cerr << "Indexations happened: " << csDefaultControllerGuard->GetInsertStartedCounter().Val() << Endl; Cerr << "Cleanups happened: " << csDefaultControllerGuard->GetCleaningStartedCounter().Val() << Endl; Cerr << "Old portions: " << JoinStrings(oldPortions.begin(), oldPortions.end(), " ") << Endl; Cerr << "Cleaned up portions: " << JoinStrings(deletedPortions.begin(), deletedPortions.end(), " ") << Endl; @@ -2705,7 +2693,8 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { } // Advance the time and trigger some more cleanups withno compactions - csDefaultControllerGuard->SetOverrideUsedSnapshotLivetime(csDefaultControllerGuard->GetMaxReadStalenessInMem() - TDuration::MilliSeconds(1)); + csDefaultControllerGuard->SetOverrideUsedSnapshotLivetime( + csDefaultControllerGuard->GetMaxReadStalenessInMem() - TDuration::MilliSeconds(1)); csDefaultControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); { auto read = std::make_unique(); @@ -2718,7 +2707,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { planStep = ProposeCommit(runtime, sender, txId, writeIds); PlanCommit(runtime, sender, planStep, txId); } -// UNIT_ASSERT_EQUAL(cleanupsHappened, 0); + // UNIT_ASSERT_EQUAL(cleanupsHappened, 0); csDefaultControllerGuard->SetOverrideStalenessLivetimePing(TDuration::Zero()); csDefaultControllerGuard->SetOverrideUsedSnapshotLivetime(TDuration::Zero()); { @@ -2737,7 +2726,6 @@ Y_UNIT_TEST_SUITE(TColumnShardTestReadWrite) { csDefaultControllerGuard->SetOverrideMaxReadStaleness(TDuration::Zero()); csDefaultControllerGuard->WaitCleaning(TDuration::Seconds(20), &runtime); Cerr << "Compactions happened: " << csDefaultControllerGuard->GetCompactionStartedCounter().Val() << Endl; - Cerr << "Indexations happened: " << csDefaultControllerGuard->GetInsertStartedCounter().Val() << Endl; Cerr << "Cleanups happened: " << csDefaultControllerGuard->GetCleaningStartedCounter().Val() << Endl; Cerr << "Old portions: " << JoinStrings(oldPortions.begin(), oldPortions.end(), " ") << Endl; Cerr << "Cleaned up portions: " << JoinStrings(deletedPortions.begin(), deletedPortions.end(), " ") << Endl; diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index 8572144b81fc..f76305e1b4d1 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -132,7 +132,7 @@ enum class EExpectedResult { static constexpr ui32 PORTION_ROWS = 80 * 1000; -void TestTtl(bool reboots, bool internal, bool useFirstPkColumnForTtl, bool writePortionsOnInsert, NScheme::TTypeId ttlColumnTypeId) +void TestTtl(bool reboots, bool internal, bool useFirstPkColumnForTtl, NScheme::TTypeId ttlColumnTypeId) { auto csControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); csControllerGuard->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); @@ -162,7 +162,6 @@ void TestTtl(bool reboots, bool internal, bool useFirstPkColumnForTtl, bool writ TTestBasicRuntime runtime; TTester::Setup(runtime); - runtime.GetAppData().FeatureFlags.SetEnableWritePortionsOnInsert(writePortionsOnInsert); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_TRACE); TActorId sender = runtime.AllocateEdgeActor(); @@ -185,7 +184,7 @@ void TestTtl(bool reboots, bool internal, bool useFirstPkColumnForTtl, bool writ const auto ttlAllDataStale = TDuration::Seconds(now - lastTs) - TDuration::Days(1); - auto spec = TTestSchema::TTableSpecials{}.WithForcedCompaction(writePortionsOnInsert); + auto spec = TTestSchema::TTableSpecials{}.WithForcedCompaction(true); spec.TtlColumn = ttlColumnName; spec.EvictAfter = ttlAllDataFresh; auto planStep = SetupSchema(runtime, sender, TTestSchema::CreateInitShardTxBody(tableId, ydbSchema, ydbPk, spec, "/Root/olapStore"), ++txId); @@ -1124,10 +1123,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { } } - Y_UNIT_TEST_SEDECIM(TTL, Reboot, Internal, FirstPkColumn, WritePortionsOnInsert) { + Y_UNIT_TEST_OCTO(TTL, Reboot, Internal, FirstPkColumn) { for (auto typeId : { NTypeIds::Timestamp, NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64 }) { Cerr << "Running TestTtl ttlColumnType=" << NKikimr::NScheme::TypeName(typeId) << Endl; - TestTtl(Reboot, Internal, FirstPkColumn, WritePortionsOnInsert, typeId); + TestTtl(Reboot, Internal, FirstPkColumn, typeId); } } diff --git a/ydb/core/tx/data_events/write_data.cpp b/ydb/core/tx/data_events/write_data.cpp index f3779a1cb5af..919f168bc782 100644 --- a/ydb/core/tx/data_events/write_data.cpp +++ b/ydb/core/tx/data_events/write_data.cpp @@ -8,12 +8,11 @@ namespace NKikimr::NEvWrite { TWriteData::TWriteData(const std::shared_ptr& writeMeta, IDataContainer::TPtr data, const std::shared_ptr& primaryKeySchema, - const std::shared_ptr& blobsAction, const bool writePortions) + const std::shared_ptr& blobsAction) : WriteMeta(writeMeta) , Data(data) , PrimaryKeySchema(primaryKeySchema) - , BlobsAction(blobsAction) - , WritePortions(writePortions) { + , BlobsAction(blobsAction) { AFL_VERIFY(WriteMeta); Y_ABORT_UNLESS(Data); Y_ABORT_UNLESS(PrimaryKeySchema); diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index 0f005acb826d..409d48fd247d 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -107,11 +107,10 @@ class TWriteData { YDB_READONLY_DEF(std::shared_ptr, PrimaryKeySchema); YDB_READONLY_DEF(std::shared_ptr, BlobsAction); YDB_ACCESSOR_DEF(std::optional, SchemaSubset); - YDB_READONLY(bool, WritePortions, false); public: TWriteData(const std::shared_ptr& writeMeta, IDataContainer::TPtr data, const std::shared_ptr& primaryKeySchema, - const std::shared_ptr& blobsAction, const bool writePortions); + const std::shared_ptr& blobsAction); const NArrow::TSchemaSubset& GetSchemaSubsetVerified() const { AFL_VERIFY(SchemaSubset); diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index c39db923f860..77e001447e76 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -940,7 +940,6 @@ Y_UNIT_TEST_SUITE(TOlap) { planStep = NTxUT::ProposeCommit(runtime, sender, shardId, txId, writeIds, txId); NTxUT::PlanCommit(runtime, sender, shardId, planStep, { txId }); } - csController->WaitIndexation(TDuration::Seconds(5)); { auto description = DescribePrivatePath(runtime, TTestTxConfig::SchemeShard, "/MyRoot/OlapStore", true, true); Cerr << description.DebugString() << Endl; From b39d884d9918c303d7b52147e967ee1f353f48a1 Mon Sep 17 00:00:00 2001 From: xyliganSereja Date: Sat, 14 Jun 2025 00:40:17 +0300 Subject: [PATCH 19/62] add view for errors #18396 (#19468) --- .../blobs_action/storages_manager/manager.cpp | 3 +- .../columnshard/blobs_action/tier/adapter.cpp | 31 +++-- .../columnshard/blobs_action/tier/adapter.h | 52 ++++++--- .../columnshard/blobs_action/tier/storage.cpp | 14 ++- .../columnshard/blobs_action/tier/storage.h | 25 ++-- ydb/core/tx/columnshard/columnshard_view.cpp | 107 ++++++++++++++++-- .../columnshard/counters/error_collector.cpp | 5 + .../tx/columnshard/counters/error_collector.h | 67 +++++++++++ ydb/core/tx/columnshard/counters/indexation.h | 2 + ydb/core/tx/columnshard/counters/ya.make | 1 + .../export/session/storage/s3/storage.cpp | 19 +++- .../tx/columnshard/test_helper/helper.cpp | 6 +- 12 files changed, 279 insertions(+), 53 deletions(-) create mode 100644 ydb/core/tx/columnshard/counters/error_collector.cpp create mode 100644 ydb/core/tx/columnshard/counters/error_collector.h diff --git a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp index bd2a8acec539..1d4dab56f440 100644 --- a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp @@ -26,7 +26,8 @@ std::shared_ptr TStoragesManager::DoBuild } return std::make_shared(storageId, Shard.SelfId(), std::make_shared("fakeBucket", "fakeSecret"), - SharedBlobsManager->GetStorageManagerGuarantee(storageId), Shard.Executor()->Generation()); + SharedBlobsManager->GetStorageManagerGuarantee(storageId), Shard.Executor()->Generation(), + Shard.Counters.GetEvictionCounters().TieringErrors); #else return nullptr; #endif diff --git a/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp b/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp index 8d42d230ee06..463c5bd90ae2 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/adapter.cpp @@ -1,12 +1,15 @@ #include "adapter.h" -#include + #include +#include #include #include +#include namespace NKikimr::NOlap::NBlobOperations::NTier { -std::unique_ptr TRepliesAdapter::RebuildReplyEvent(std::unique_ptr&& ev) const { +std::unique_ptr TRepliesAdapter::RebuildReplyEvent( + std::unique_ptr&& ev) const { TLogoBlobID logoBlobId; TString error; AFL_VERIFY(TLogoBlobID::Parse(logoBlobId, *ev->Key, error))("error", error)("str_blob_id", *ev->Key); @@ -16,26 +19,38 @@ std::unique_ptr TRepliesAdapter::RebuildReplyEvent(std::uni } if (ev->IsSuccess()) { AFL_VERIFY(!!ev->Body)("key", ev->Key)("interval_from", ev->GetReadInterval().first)("interval_to", ev->GetReadInterval().second); - return std::make_unique(bRange, NKikimrProto::EReplyStatus::OK, ev->Body, TString{}, false, StorageId); + return std::make_unique( + bRange, NKikimrProto::EReplyStatus::OK, ev->Body, TString{}, false, StorageId); } else { AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "s3_request_failed")("request_type", "get_object")( "exception", ev->GetError().GetExceptionName())("message", ev->GetError().GetMessage())("storage_id", StorageId)("blob", logoBlobId); - return std::make_unique(bRange, NKikimrProto::EReplyStatus::ERROR, TStringBuilder() << ev->Result, TStringBuilder{} << ev->GetError().GetExceptionName() << ", " << ev->GetError().GetMessage(), false, StorageId); + TString err = TStringBuilder() << ev->GetError().GetExceptionName() << ", " << ev->GetError().GetMessage(); + ErrorCollector->OnReadError(StorageId, err); + + return std::make_unique(bRange, NKikimrProto::EReplyStatus::ERROR, + TStringBuilder() << ev->Result, err, false, + StorageId); } } -std::unique_ptr TRepliesAdapter::RebuildReplyEvent(std::unique_ptr&& ev) const { +std::unique_ptr TRepliesAdapter::RebuildReplyEvent( + std::unique_ptr&& ev) const { TLogoBlobID logoBlobId; TString error; Y_ABORT_UNLESS(ev->Key); AFL_VERIFY(TLogoBlobID::Parse(logoBlobId, *ev->Key, error))("error", error)("str_blob_id", *ev->Key); if (ev->IsSuccess()) { - return std::make_unique(NKikimrProto::EReplyStatus::OK, logoBlobId, 0, TGroupId::FromValue(Max()), 0, StorageId); + return std::make_unique( + NKikimrProto::EReplyStatus::OK, logoBlobId, 0, TGroupId::FromValue(Max()), 0, StorageId); } else { AFL_DEBUG(NKikimrServices::TX_TIERING)("event", "s3_request_failed")("request_type", "put_object")( "exception", ev->GetError().GetExceptionName())("message", ev->GetError().GetMessage())("storage_id", StorageId)("blob", logoBlobId); - return std::make_unique(NKikimrProto::EReplyStatus::ERROR, logoBlobId, 0, TGroupId::FromValue(Max()), 0, StorageId); + TString err = TStringBuilder() << ev->GetError().GetExceptionName() << ", " << ev->GetError().GetMessage(); + ErrorCollector->OnWriteError(StorageId, err); + + return std::make_unique( + NKikimrProto::EReplyStatus::ERROR, logoBlobId, 0, TGroupId::FromValue(Max()), 0, StorageId); } } -} +} // namespace NKikimr::NOlap::NBlobOperations::NTier diff --git a/ydb/core/tx/columnshard/blobs_action/tier/adapter.h b/ydb/core/tx/columnshard/blobs_action/tier/adapter.h index 4e146e019137..5c65ebf4b872 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/adapter.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/adapter.h @@ -1,59 +1,77 @@ #pragma once + +#include +#include #include +#include + namespace NKikimr::NOlap::NBlobOperations::NTier { class TRepliesAdapter: public NWrappers::NExternalStorage::IReplyAdapter { private: + const std::shared_ptr ErrorCollector; const TString StorageId; -public: - TRepliesAdapter(const TString& storageId) - : StorageId(storageId) - { +public: + TRepliesAdapter(const std::shared_ptr& errorCollector, const TString& storageId) + : ErrorCollector(std::move(errorCollector)) + , StorageId(std::move(storageId)) { } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override; - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override; - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override; + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override; + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { return std::move(ev); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } - virtual std::unique_ptr RebuildReplyEvent(std::unique_ptr&& ev) const override { + virtual std::unique_ptr RebuildReplyEvent( + std::unique_ptr&& ev) const override { Y_UNUSED(ev); Y_ABORT_UNLESS(false); } }; -} +} // namespace NKikimr::NOlap::NBlobOperations::NTier diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index 4b63d1316362..a6bbc90f60d0 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -7,6 +7,7 @@ #include "write.h" #include +#include #include #include @@ -17,7 +18,8 @@ NWrappers::NExternalStorage::IExternalStorageOperator::TPtr TOperator::GetCurren return ExternalStorageOperator->Get(); } -std::shared_ptr TOperator::DoStartDeclareRemovingAction(const std::shared_ptr& counters) { +std::shared_ptr TOperator::DoStartDeclareRemovingAction( + const std::shared_ptr& counters) { return std::make_shared(GetStorageId(), GetSelfTabletId(), counters, GCInfo); } @@ -85,17 +87,19 @@ void TOperator::InitNewExternalOperator() { void TOperator::DoInitNewExternalOperator(const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& storageOperator, const std::optional& settings) { - storageOperator->InitReplyAdapter(std::make_shared(GetStorageId())); + storageOperator->InitReplyAdapter(std::make_shared(ErrorCollector, GetStorageId())); { TGuard changeLock(ChangeOperatorLock); CurrentS3Settings = settings; } + ExternalStorageOperator->Emplace(storageOperator); } TOperator::TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, const std::shared_ptr& storageSharedBlobsManager) : TBase(storageId, storageSharedBlobsManager) + , ErrorCollector(shard.Counters.GetEvictionCounters().TieringErrors) , TabletActorId(shard.SelfId()) , Generation(shard.Executor()->Generation()) , ExternalStorageOperator(std::make_shared()) { @@ -104,8 +108,10 @@ TOperator::TOperator(const TString& storageId, const NColumnShard::TColumnShard& TOperator::TOperator(const TString& storageId, const TActorId& shardActorId, const std::shared_ptr& storageConfig, - const std::shared_ptr& storageSharedBlobsManager, const ui64 generation) + const std::shared_ptr& storageSharedBlobsManager, const ui64 generation, + const std::shared_ptr& errorCollector) : TBase(storageId, storageSharedBlobsManager) + , ErrorCollector(std::move(errorCollector)) , TabletActorId(shardActorId) , Generation(generation) , InitializationConfig(storageConfig) @@ -129,4 +135,4 @@ bool TOperator::DoLoad(IBlobManagerDb& dbBlobs) { return true; } -} +} // namespace NKikimr::NOlap::NBlobOperations::NTier diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.h b/ydb/core/tx/columnshard/blobs_action/tier/storage.h index 2cad89fa3816..2dec0447db0c 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.h @@ -1,17 +1,20 @@ #pragma once -#include +#include "gc_info.h" + #include +#include #include +#include +#include #include #include -#include "gc_info.h" - namespace NKikimr::NOlap::NBlobOperations::NTier { class TOperator: public IBlobsStorageOperator { private: using TBase = IBlobsStorageOperator; + const std::shared_ptr ErrorCollector; const NActors::TActorId TabletActorId; const ui64 Generation; std::shared_ptr GCInfo = std::make_shared(); @@ -26,14 +29,16 @@ class TOperator: public IBlobsStorageOperator { TAtomicCounter StepCounter; void InitNewExternalOperator(const NColumnShard::NTiers::TManager* tierManager); void InitNewExternalOperator(); - void DoInitNewExternalOperator( - const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& storageOperator, const std::optional& settings); + void DoInitNewExternalOperator(const NWrappers::NExternalStorage::IExternalStorageOperator::TPtr& storageOperator, + const std::optional& settings); virtual TString DoDebugString() const override { return GetCurrentOperator()->DebugString(); } + protected: - virtual std::shared_ptr DoStartDeclareRemovingAction(const std::shared_ptr& counters) override; + virtual std::shared_ptr DoStartDeclareRemovingAction( + const std::shared_ptr& counters) override; virtual std::shared_ptr DoStartWritingAction() override; virtual std::shared_ptr DoStartReadingAction() override; virtual std::shared_ptr DoCreateGCAction(const std::shared_ptr& counters) const override; @@ -42,9 +47,11 @@ class TOperator: public IBlobsStorageOperator { virtual void DoOnTieringModified(const std::shared_ptr& tiers) override; public: - TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, const std::shared_ptr& storageSharedBlobsManager); + TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, + const std::shared_ptr& storageSharedBlobsManager); TOperator(const TString& storageId, const TActorId& shardActorId, const std::shared_ptr& storageConfig, - const std::shared_ptr& storageSharedBlobsManager, const ui64 generation); + const std::shared_ptr& storageSharedBlobsManager, const ui64 generation, + const std::shared_ptr& errorCollector); virtual TTabletsByBlob GetBlobsToDelete() const override { auto result = GCInfo->GetBlobsToDelete(); @@ -65,4 +72,4 @@ class TOperator: public IBlobsStorageOperator { } }; -} +} // namespace NKikimr::NOlap::NBlobOperations::NTier diff --git a/ydb/core/tx/columnshard/columnshard_view.cpp b/ydb/core/tx/columnshard/columnshard_view.cpp index 3df6694977d9..f117ec6358bc 100644 --- a/ydb/core/tx/columnshard/columnshard_view.cpp +++ b/ydb/core/tx/columnshard/columnshard_view.cpp @@ -3,12 +3,12 @@ namespace NKikimr::NColumnShard { -class TTxMonitoring : public TTransactionBase { +class TTxMonitoring: public TTransactionBase { public: TTxMonitoring(TColumnShard* self, const NMon::TEvRemoteHttpInfo::TPtr& ev) : TBase(self) - , HttpInfoEvent(ev) - {} + , HttpInfoEvent(ev) { + } bool Execute(TTransactionContext& txc, const TActorContext& ctx) override; void Complete(const TActorContext& ctx) override; @@ -19,15 +19,97 @@ class TTxMonitoring : public TTransactionBase { NJson::TJsonValue JsonReport = NJson::JSON_MAP; }; +inline TString TEscapeHtml(const TString& in) { + TString out; + out.reserve(in.size()); + for (char c : in) { + switch (c) { + case '<': + out += "<"; + break; + case '>': + out += ">"; + break; + case '&': + out += "&"; + break; + case '\'': + out += "'"; + break; + case '\"': + out += """; + break; + default: + out += c; + break; + } + } + + return out; +} bool TTxMonitoring::Execute(TTransactionContext& txc, const TActorContext&) { return Self->TablesManager.FillMonitoringReport(txc, JsonReport["tables_manager"]); } +template +void TPrintErrorTable(TStringStream& html, THashMap> errors, const std::string& errorType) { + html << R"( + + )"; + + if (errors.empty()) { + html << "No " << errorType << " errors
"; + } else { + html << "" << errorType << " errors:
"; + html << "" + ""; + + for (auto [tier, queue] : errors) { + while (!queue.empty()) { + const auto& element = queue.front(); + html << "" + << "" + << "" + << "" + << ""; + queue.pop(); + } + } + + html << "
TierTimeError
" << TEscapeHtml(tier) << "" << element.Time.ToString() << "" << TEscapeHtml(element.Reason) << "

"; + } +} + void TTxMonitoring::Complete(const TActorContext& ctx) { const auto& cgi = HttpInfoEvent->Get()->Cgi(); std::map, NJson::TJsonValue> schemaVersions; - for (const auto& item: JsonReport["tables_manager"]["schema_versions"].GetArray()) { + for (const auto& item : JsonReport["tables_manager"]["schema_versions"].GetArray()) { auto& schemaVersion = schemaVersions[std::make_pair(item["SinceStep"].GetInteger(), item["SinceTxId"].GetInteger())]; schemaVersion = item; } @@ -43,7 +125,8 @@ void TTxMonitoring::Complete(const TActorContext& ctx) { TStringStream html; html << "

Special Values

"; html << "CurrentSchemeShardId: " << Self->CurrentSchemeShardId << "
"; - html << "ProcessingParams: " << Self->ProcessingParams.value_or(NKikimrSubDomains::TProcessingParams{}).ShortDebugString() << "
"; + html << "ProcessingParams: " << Self->ProcessingParams.value_or(NKikimrSubDomains::TProcessingParams{}).ShortDebugString() + << "
"; html << "LastPlannedStep: " << Self->LastPlannedStep << "
"; html << "LastPlannedTxId : " << Self->LastPlannedTxId << "
"; html << "LastSchemaSeqNoGeneration : " << Self->LastSchemaSeqNo.Generation << "
"; @@ -61,19 +144,27 @@ void TTxMonitoring::Complete(const TActorContext& ctx) { html << "
" << JsonReport << "

"; html << "

Top " << countVersions << " of " << schemaVersions.size() << " Versions

"; size_t counter = 0; - for (const auto& [_, schemaVersion]: schemaVersions) { + for (const auto& [_, schemaVersion] : schemaVersions) { html << counter; html << "
" << schemaVersion << "

"; if (++counter == countVersions) { break; } } + + html << "

Tiering Errors

"; + auto readErrors = Self->Counters.GetEvictionCounters().TieringErrors->GetAllReadErrors(); + auto writeErrors = Self->Counters.GetEvictionCounters().TieringErrors->GetAllWriteErrors(); + + TPrintErrorTable(html, readErrors, "read"); + TPrintErrorTable(html, writeErrors, "write"); + ctx.Send(HttpInfoEvent->Sender, new NMon::TEvRemoteHttpInfoRes(html.Str())); } bool TColumnShard::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext& ctx) { if (!Executor() || !Executor()->GetStats().IsActive) { - return false; + return false; } if (!ev) { @@ -84,4 +175,4 @@ bool TColumnShard::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const T return true; } -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/error_collector.cpp b/ydb/core/tx/columnshard/counters/error_collector.cpp new file mode 100644 index 000000000000..3b5b7a111983 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/error_collector.cpp @@ -0,0 +1,5 @@ +#include "error_collector.h" + +namespace NKikimr::NColumnShard { + +} // namespace NKikimr::NColumnShard \ No newline at end of file diff --git a/ydb/core/tx/columnshard/counters/error_collector.h b/ydb/core/tx/columnshard/counters/error_collector.h new file mode 100644 index 000000000000..890f0ce7af10 --- /dev/null +++ b/ydb/core/tx/columnshard/counters/error_collector.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NColumnShard { + +class TPerTierErrorCollector { +public: + struct TError { + TString Reason; + TInstant Time; + }; + + TPerTierErrorCollector() { + } + + void Add(const TString& tier, const TString& reason) { + TGuard lock(Lock); + auto& q = ErrorsCollector[tier]; + if (q.size() == QUEUE_MAX_SIZE) { + q.pop(); + } + + q.push({ reason, TInstant::Now() }); + } + + THashMap> GetAll() const { + TGuard lock(Lock); + return ErrorsCollector; + } + +private: + static constexpr size_t QUEUE_MAX_SIZE = 10; + mutable TSpinLock Lock; + THashMap> ErrorsCollector; +}; + +class TErrorCollector { +public: + void OnReadError(const TString& tier, const TString& message) { + Read.Add(tier, message); + } + + void OnWriteError(const TString& tier, const TString& message) { + Write.Add(tier, message); + } + + THashMap> GetAllReadErrors() const { + return Read.GetAll(); + } + + THashMap> GetAllWriteErrors() const { + return Write.GetAll(); + } + +private: + TPerTierErrorCollector Read; + TPerTierErrorCollector Write; +}; + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/indexation.h b/ydb/core/tx/columnshard/counters/indexation.h index 5fc395e92f48..c1469998a0f4 100644 --- a/ydb/core/tx/columnshard/counters/indexation.h +++ b/ydb/core/tx/columnshard/counters/indexation.h @@ -1,4 +1,5 @@ #pragma once +#include "error_collector.h" #include "splitter.h" #include "sub_columns.h" @@ -18,6 +19,7 @@ class TIndexationCounters: public TCommonCountersOwner { NMonitoring::TDynamicCounters::TCounterPtr CompactionHugePartsCount; public: + const std::shared_ptr TieringErrors = std::make_shared(); std::shared_ptr SubColumnCounters; NMonitoring::TDynamicCounters::TCounterPtr CompactionInputBytes; diff --git a/ydb/core/tx/columnshard/counters/ya.make b/ydb/core/tx/columnshard/counters/ya.make index 1366b4cdb0e6..d62eee17f870 100644 --- a/ydb/core/tx/columnshard/counters/ya.make +++ b/ydb/core/tx/columnshard/counters/ya.make @@ -16,6 +16,7 @@ SRCS( writes_monitor.cpp portion_index.cpp duplicate_filtering.cpp + error_collector.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/export/session/storage/s3/storage.cpp b/ydb/core/tx/columnshard/export/session/storage/s3/storage.cpp index 694772fb5f06..ccd595631995 100644 --- a/ydb/core/tx/columnshard/export/session/storage/s3/storage.cpp +++ b/ydb/core/tx/columnshard/export/session/storage/s3/storage.cpp @@ -1,26 +1,35 @@ #include "storage.h" + #include #ifndef KIKIMR_DISABLE_S3_OPS #include #include #endif +#include #include +namespace { +static std::shared_ptr DummyCollector = std::make_shared(); +} + namespace NKikimr::NOlap::NExport { -NKikimr::TConclusion> TS3StorageInitializer::DoInitializeOperator(const std::shared_ptr& storages) const { +NKikimr::TConclusion> TS3StorageInitializer::DoInitializeOperator( + const std::shared_ptr& storages) const { #ifndef KIKIMR_DISABLE_S3_OPS auto extStorageConfig = NWrappers::NExternalStorage::IExternalStorageConfig::Construct(S3Settings); if (!extStorageConfig) { return TConclusionStatus::Fail("cannot build operator with this config: " + S3Settings.DebugString()); } - return std::shared_ptr(new NBlobOperations::NTier::TOperator("__EXPORT:" + StorageName, NActors::TActorId(), extStorageConfig, - std::make_shared("__EXPORT:" + StorageName, storages->GetSharedBlobsManager()->GetSelfTabletId()), - storages->GetGeneration())); + return std::shared_ptr( + new NBlobOperations::NTier::TOperator("__EXPORT:" + StorageName, NActors::TActorId(), extStorageConfig, + std::make_shared( + "__EXPORT:" + StorageName, storages->GetSharedBlobsManager()->GetSelfTabletId()), + storages->GetGeneration(), DummyCollector)); #else Y_UNUSED(storages); return TConclusionStatus::Fail("s3 not supported"); #endif } -} \ No newline at end of file +} // namespace NKikimr::NOlap::NExport diff --git a/ydb/core/tx/columnshard/test_helper/helper.cpp b/ydb/core/tx/columnshard/test_helper/helper.cpp index d64003c5d525..8bfaf75a7a3d 100644 --- a/ydb/core/tx/columnshard/test_helper/helper.cpp +++ b/ydb/core/tx/columnshard/test_helper/helper.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -16,6 +17,9 @@ #include #endif +namespace { +static std::shared_ptr DummyCollector = std::make_shared(); +} namespace NKikimr::NArrow::NTest { NKikimrSchemeOp::TOlapColumnDescription TTestColumn::CreateColumn(const ui32 id) const { @@ -110,7 +114,7 @@ std::shared_ptr TTestStoragesManager::DoB Singleton()->SetSecretKey("fakeSecret"); return std::make_shared(storageId, NActors::TActorId(), std::make_shared("fakeBucket", "fakeSecret"), - SharedBlobsManager->GetStorageManagerGuarantee(storageId), GetGeneration()); + SharedBlobsManager->GetStorageManagerGuarantee(storageId), GetGeneration(), DummyCollector); #endif } return nullptr; From 0720c839ff350d559b4079a9b6a75b42142a668e Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Sat, 14 Jun 2025 20:03:55 +0300 Subject: [PATCH 20/62] generate internal path id (#19660) --- ydb/core/protos/config.proto | 1 + ydb/core/tx/columnshard/columnshard_impl.cpp | 2 +- ydb/core/tx/columnshard/columnshard_schema.h | 3 +- .../tx/columnshard/hooks/abstract/abstract.h | 4 -- .../columnshard/hooks/testing/ro_controller.h | 3 -- ydb/core/tx/columnshard/tables_manager.cpp | 32 +++++++++------- ydb/core/tx/columnshard/tables_manager.h | 3 +- .../ut_schema/ut_columnshard_schema.cpp | 37 ++++++++++++++----- 8 files changed, 53 insertions(+), 32 deletions(-) diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 8943984e1d5d..eec56126ac5e 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1920,6 +1920,7 @@ message TColumnShardConfig { optional EJsonDoubleOutOfRangeHandlingPolicy DoubleOutOfRangeHandling = 40 [default = REJECT]; optional bool PortionMetaV0Usage = 41 [default = true]; optional uint32 PeriodicWakeupActivationPeriodMs = 42 [default = 60000]; + optional bool GenerateInternalPathId = 43 [default = false]; //Be aware! IRREVERSIBLE(if set, can not be unset for a dababase) } message TSchemeShardConfig { diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 53a1372b4c57..7437b028f606 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -292,7 +292,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl } const auto internalPathId = TablesManager.CreateInternalPathId(schemeShardLocalPathId); - LOG_S_DEBUG("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId) + LOG_S_INFO("EnsureTable for pathId: " << TUnifiedPathId(internalPathId, schemeShardLocalPathId) << " ttl settings: " << tableProto.GetTtlSettings() << " at tablet " << TabletID()); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index d0551ddccf44..6be107778adf 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -83,7 +83,8 @@ struct Schema : NIceDb::Schema { GCBarrierPreparationGen = 16, GCBarrierPreparationStep = 17, SubDomainLocalPathId = 18, - SubDomainOutOfSpace = 19 + SubDomainOutOfSpace = 19, + MaxInternalPathId = 20, //max internal path id ever known in this tablet }; enum class EInsertTableIds : ui8 { diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index dbb538daff8e..490382372c7e 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -357,10 +357,6 @@ class ICSController { return {}; } - virtual ui64 GetInternalPathIdOffset(const ui64 /*tabletId*/) const { - return 0; - } - virtual void OnAddPathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { } virtual void OnDeletePathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 851e6566b084..8646d3494051 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -179,9 +179,6 @@ class TReadOnlyController: public ICSController { } } - virtual ui64 GetInternalPathIdOffset(const ui64 tabletId) const override { - return 1000000 + static_cast(tabletId) % 1000; - } }; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 08ba00ef3ee4..c1f33ce06e7b 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -64,6 +64,15 @@ bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionConte } bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { + { + std::optional maxPathId; + if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::MaxInternalPathId, maxPathId)) { + return false; + } + if (maxPathId) { + MaxInternalPathId =TInternalPathId::FromRawValue(*maxPathId); + } + } { TLoadTimeSignals::TLoadTimer timer = LoadTimeCounters->TableLoadTimeCounters.StartGuard(); TMemoryProfileGuard g("TTablesManager/InitFromDB::Tables"); @@ -79,6 +88,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { AFL_VERIFY(PathsToDrop[table.GetDropVersionVerified()].emplace(table.GetPathId().InternalPathId).second); } const auto& pathId = table.GetPathId(); + AFL_VERIFY(pathId.InternalPathId <= MaxInternalPathId)("path_id", pathId)("max_internal_path_id", MaxInternalPathId); AFL_VERIFY(Tables.emplace(pathId.InternalPathId, std::move(table)).second); AFL_VERIFY(SchemeShardLocalToInternal.emplace(pathId.SchemeShardLocalPathId, pathId.InternalPathId).second); @@ -226,9 +236,13 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet } TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) { - const auto result = TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue() + InternalPathIdOffset); - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("method", "CreateInternalPathId")("schemeShardLocalPathId", schemeShardLocalPathId)("result", result); - return result; + if (AppData()->ColumnShardConfig.GetGenerateInternalPathId()) { + const auto result = TInternalPathId::FromRawValue(MaxInternalPathId.GetRawValue() + 1); + MaxInternalPathId = result; + return result; + } else { + return TInternalPathId::FromRawValue(schemeShardLocalPathId.GetRawValue()); + } } bool TTablesManager::IsReadyForStartWrite(const TInternalPathId pathId, const bool withDeleted) const { @@ -281,6 +295,7 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size()); AFL_VERIFY(SchemeShardLocalToInternal.emplace(table.GetPathId().SchemeShardLocalPathId, table.GetPathId().InternalPathId).second); Schema::SaveTableSchemeShardLocalPathId(db, table.GetPathId().InternalPathId, table.GetPathId().SchemeShardLocalPathId); + Schema::SaveSpecialValue(db, Schema::EValueIds::MaxInternalPathId, MaxInternalPathId.GetRawValue()); if (PrimaryIndex) { PrimaryIndex->RegisterTable(pathId); } @@ -369,14 +384,6 @@ void TTablesManager::AddTableVersion(const TInternalPathId pathId, const NOlap:: table.AddVersion(version); } -namespace { - -ui64 GetInternalPathIdOffset(ui64 tabletId) { - return NYDBTest::TControllers::GetColumnShardController()->GetInternalPathIdOffset(tabletId); -} - -} //namespace - TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const std::shared_ptr& dataAccessorsManager, const std::shared_ptr& schemaCache, const std::shared_ptr& portionsStats, @@ -386,8 +393,7 @@ TTablesManager::TTablesManager(const std::shared_ptr& s , LoadTimeCounters(std::make_unique()) , SchemaObjectsCache(schemaCache) , PortionsStats(portionsStats) - , TabletId(tabletId) - , InternalPathIdOffset(GetInternalPathIdOffset(tabletId)) { + , TabletId(tabletId) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const TInternalPathId pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 0d2ef20f8659..79a9ced3945a 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -210,7 +210,8 @@ class TTablesManager: public NOlap::IPathIdTranslator { NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; ui64 TabletId = 0; - ui64 InternalPathIdOffset; + static constexpr ui64 MaxInternalPathIdDefault = 1'000'000'000; //Use a value presumably greater than any really used + TInternalPathId MaxInternalPathId = TInternalPathId::FromRawValue(MaxInternalPathIdDefault); //Max internal path id ever used in this tablet friend class TTxInit; diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index f76305e1b4d1..cc232fa70911 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -865,9 +865,10 @@ void TestExport(bool reboot, TExportTestOpts&& opts = TExportTestOpts{}) { } } -void TestDrop(bool reboots) { +void TestDrop(bool reboots, bool generateInternalPathId) { TTestBasicRuntime runtime; TTester::Setup(runtime); + runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); TActorId sender = runtime.AllocateEdgeActor(); @@ -1035,7 +1036,7 @@ extern bool gAllowLogBatchingDefaultValue; Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { - Y_UNIT_TEST(CreateTable) { + void CreateTable(bool reboots, bool generateInternalPathId) { ui64 tableId = 1; std::vector intTypes = { @@ -1058,6 +1059,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { TTestBasicRuntime runtime; TTester::Setup(runtime); + runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); using namespace NTxUT; @@ -1084,6 +1086,24 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { planStep = SetupSchema(runtime, sender, txBody, txId++); } + { + const auto& pathIdTranslator = csDefaultControllerGuard->GetPathIdTranslator(TTestTxConfig::TxTablet0); + const auto pathIds = pathIdTranslator->GetSchemeShardLocalPathIds(); + for (const auto& pathId : pathIds) { + const auto& internalPathId = pathIdTranslator->ResolveInternalPathId(pathId); + UNIT_ASSERT(internalPathId); + if (generateInternalPathId) { + UNIT_ASSERT_VALUES_UNEQUAL(internalPathId->GetRawValue(), pathId.GetRawValue()); + } else { + UNIT_ASSERT_VALUES_EQUAL(internalPathId->GetRawValue(), pathId.GetRawValue()); + } + } + } + + if (reboots) { + RebootTablet(runtime, TTestTxConfig::TxTablet0, sender); + } + // TODO: support float types std::vector floatTypes = { NTypeIds::Float, @@ -1123,6 +1143,10 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { } } + Y_UNIT_TEST_QUATRO(CreateTable, Reboots, GenerateInternalPathId) { + CreateTable(Reboots, GenerateInternalPathId); + } + Y_UNIT_TEST_OCTO(TTL, Reboot, Internal, FirstPkColumn) { for (auto typeId : { NTypeIds::Timestamp, NTypeIds::Datetime, NTypeIds::Date, NTypeIds::Uint32, NTypeIds::Uint64 }) { Cerr << "Running TestTtl ttlColumnType=" << NKikimr::NScheme::TypeName(typeId) << Endl; @@ -1236,14 +1260,9 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { TestCompaction(); } - Y_UNIT_TEST(Drop) { - TestDrop(false); + Y_UNIT_TEST_QUATRO(Drop, Reboots, GenerateInternalPathId) { + TestDrop(Reboots, GenerateInternalPathId); } - - Y_UNIT_TEST(RebootDrop) { - TestDrop(true); - } - Y_UNIT_TEST(DropWriteRace) { TestDropWriteRace(); } From b8440b0b2482e5508c2d670b6a01373b76e329c0 Mon Sep 17 00:00:00 2001 From: Vladilen Date: Mon, 16 Jun 2025 10:11:45 +0200 Subject: [PATCH 21/62] Add ColumnShard write statistics (#19642) --- .github/config/muted_ya.txt | 4 + ydb/core/kqp/ut/cost/kqp_cost_ut.cpp | 218 +++++++++++++----- .../blobs_action/common/statistics.cpp | 16 ++ .../blobs_action/common/statistics.h | 12 + .../columnshard/blobs_action/common/ya.make | 2 + .../transaction/tx_blobs_written.cpp | 3 + 6 files changed, 195 insertions(+), 60 deletions(-) create mode 100644 ydb/core/tx/columnshard/blobs_action/common/statistics.cpp create mode 100644 ydb/core/tx/columnshard/blobs_action/common/statistics.h diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index d8a863f95c79..a88664d46b15 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -26,6 +26,10 @@ ydb/core/kqp/ut/olap KqpOlapSysView.StatsSysViewBytesDictActualization ydb/core/kqp/ut/olap KqpOlapSysView.StatsSysViewBytesDictStatActualization ydb/core/kqp/ut/olap KqpOlapJson.DuplicationCompactionVariants ydb/core/kqp/ut/olap KqpOlapWrite.TierDraftsGCWithRestart +ydb/core/kqp/ut/batch_operations [*/*] chunk chunk +ydb/core/kqp/ut/federated_query/large_results KqpScriptExecResults.ExecuteScriptWithLargeFile +ydb/core/kqp/ut/federated_query/s3 sole chunk chunk +ydb/core/kqp/ut/indexes KqpMultishardIndex.WriteIntoRenamingSyncIndex ydb/core/kqp/ut/olap [*/*] chunk chunk ydb/core/kqp/ut/join KqpIndexLookupJoin.LeftJoinRightNullFilter+StreamLookup ydb/core/kqp/ut/join KqpIndexLookupJoin.LeftJoinRightNullFilter-StreamLookup diff --git a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp index 9c446a473287..fadc5cd88447 100644 --- a/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp +++ b/ydb/core/kqp/ut/cost/kqp_cost_ut.cpp @@ -615,44 +615,92 @@ Y_UNIT_TEST_SUITE(KqpCost) { TKikimrRunner kikimr(GetAppConfig(false, false, true)); auto db = kikimr.GetQueryClient(); auto session = db.GetSession().GetValueSync().GetSession(); + const TString testTableName = "/Root/TestTable"; + const ui64 minRowBytes = 300; CreateTestTable(session, true); - { - auto query = Q_(R"( - REPLACE INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (1u, "Anna", 3500u, "None"); - )"); - + auto checkUpdatesStats = [&testTableName, &session](const TString& query, ui64 rows) { auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + const auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + Cerr << stats.DebugString() << Endl; - auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); + UNIT_ASSERT_VALUES_UNEQUAL(stats.query_phases_size(), 0); + const auto& phase = stats.query_phases(0); - Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().bytes(), 20); - } + UNIT_ASSERT_VALUES_EQUAL(phase.table_access_size(), 1); + const auto& tableAccess = phase.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.updates().rows(), rows); + UNIT_ASSERT_GT(tableAccess.updates().bytes(), rows * minRowBytes); + }; + + checkUpdatesStats(Q_(R"( + REPLACE INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (1u, "Anna", 3500u, "None"); + )"), 1); + + checkUpdatesStats(R"( + UPSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (1u, "Anna", 3500u, "None"); + )", 1); + + // INSERT 1 NEW ROW + checkUpdatesStats(Q_(R"( + INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (3u, "Anna", 3500u, "None"); + )"), 1); + + // INSERT 2 NEW ROWS + checkUpdatesStats(Q_(R"( + INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) + VALUES (101u, "Anna", 3500u, "None"), (102u, "Anna", 3500u, "None"); + )"), 2); + + // UPDATE + // TODO: reads ?? + checkUpdatesStats(Q_(R"( + UPDATE `/Root/TestTable` ON SELECT 3u AS Group, "Anna" AS Name, 4000u AS Amount, "None" AS Comment; + )"), 1); { + // 2 SEPARATE INSERTS auto query = Q_(R"( - UPSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (1u, "Anna", 3500u, "None"); + INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) + VALUES (401u, "Anna", 3500u, "None"); + + INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) + VALUES (402u, "Anna", 3500u, "None"); )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().bytes(), 20); + + UNIT_ASSERT_GT(stats.query_phases_size(), 1); + const auto& phase0 = stats.query_phases(0); + + UNIT_ASSERT_VALUES_EQUAL(phase0.table_access_size(), 1); + const auto& tableAccess0 = phase0.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess0.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess0.updates().rows(), 1); + UNIT_ASSERT_GT(tableAccess0.updates().bytes(), minRowBytes); + + const auto& phase1 = stats.query_phases(1); + + UNIT_ASSERT_VALUES_EQUAL(phase1.table_access_size(), 1); + const auto& tableAccess1 = phase1.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess1.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess1.updates().rows(), 1); + UNIT_ASSERT_GT(tableAccess1.updates().bytes(), minRowBytes); } { @@ -664,40 +712,54 @@ Y_UNIT_TEST_SUITE(KqpCost) { auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - // TODO: fix status? - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); Cerr << stats.DebugString() << Endl; // TODO: reads??? - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().bytes(), 8); + for (const auto& phase : stats.query_phases()) { + for (const auto& tableAccess : phase.table_access()) { + if (tableAccess.name() != testTableName) { + continue; + } + UNIT_ASSERT_VALUES_EQUAL(tableAccess.updates().rows(), 0); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.updates().bytes(), 0); + } + } } { - // INSERT NEW + // UPDATE empty auto query = Q_(R"( - INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) VALUES (3u, "Anna", 3500u, "None"); + UPDATE `/Root/TestTable` ON SELECT 4u AS Group, "Anna" AS Name, 4000u AS Amount, "None" AS Comment; )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().bytes(), 20); + // No writes + // TODO: reads ?? + for (const auto& phase : stats.query_phases()) { + for (const auto& tableAccess : phase.table_access()) { + if (tableAccess.name() != testTableName) { + continue; + } + UNIT_ASSERT_VALUES_EQUAL(tableAccess.updates().rows(), 0); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.updates().bytes(), 0); + } + } } { - // UPDATE empty + // DELETE empty auto query = Q_(R"( - UPDATE `/Root/TestTable` ON SELECT 4u AS Group, "Anna" AS Name, 4000u AS Amount, "None" AS Comment; + DELETE FROM `/Root/TestTable` ON SELECT 4u AS Group, "Anna" AS Name; )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); @@ -708,73 +770,109 @@ Y_UNIT_TEST_SUITE(KqpCost) { auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); Cerr << stats.DebugString() << Endl; - // No reads & no writes - for (int phase = 0; phase < stats.query_phases_size(); ++phase) { - if (stats.query_phases(phase).table_access_size() > 0) { - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access_size(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().rows(), 0); - } - } + + UNIT_ASSERT_VALUES_UNEQUAL(stats.query_phases_size(), 0); + const auto& phase = stats.query_phases(0); + + UNIT_ASSERT_VALUES_EQUAL(phase.table_access_size(), 1); + const auto& tableAccess = phase.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().bytes(), 0); } { - // UPDATE + // DELETE auto query = Q_(R"( - UPDATE `/Root/TestTable` ON SELECT 3u AS Group, "Anna" AS Name, 4000u AS Amount, "None" AS Comment; + DELETE FROM `/Root/TestTable` ON SELECT 3u AS Group, "Anna" AS Name; )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).updates().bytes(), 20); - // TODO: reads??? - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(1).table_access(0).reads().bytes(), 8); + + UNIT_ASSERT_VALUES_UNEQUAL(stats.query_phases_size(), 0); + const auto& phase = stats.query_phases(0); + + UNIT_ASSERT_VALUES_EQUAL(phase.table_access_size(), 1); + const auto& tableAccess = phase.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().bytes(), 0); } { - // DELETE empty + // DELETE THEN INSERT auto query = Q_(R"( - DELETE FROM `/Root/TestTable` ON SELECT 4u AS Group, "Anna" AS Name; + DELETE FROM `/Root/TestTable` ON SELECT 1u AS Group, "Paul" AS Name; + + INSERT INTO `/Root/TestTable` (Group, Name, Amount, Comment) + VALUES (501u, "Anna", 3500u, "None"); )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).deletes().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).deletes().bytes(), 0); + + UNIT_ASSERT_GT(stats.query_phases_size(), 1); + const auto& phase0 = stats.query_phases(0); + + UNIT_ASSERT_VALUES_EQUAL(phase0.table_access_size(), 1); + const auto& tableAccess0 = phase0.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess0.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess0.deletes().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(tableAccess0.deletes().bytes(), 0); + + const auto& phase1 = stats.query_phases(1); + + UNIT_ASSERT_VALUES_EQUAL(phase1.table_access_size(), 1); + const auto& tableAccess1 = phase1.table_access(0); + + UNIT_ASSERT_VALUES_EQUAL(tableAccess1.name(), testTableName); + UNIT_ASSERT_VALUES_EQUAL(tableAccess1.updates().rows(), 1); + UNIT_ASSERT_VALUES_EQUAL(tableAccess1.updates().bytes(), 368); } { - // DELETE + // DELETE 2 ROWS + // rows are added in the 2 SEPARATE INSERTS test above auto query = Q_(R"( - DELETE FROM `/Root/TestTable` ON SELECT 3u AS Group, "Anna" AS Name; + DELETE FROM `/Root/TestTable` ON SELECT Group, Name FROM `/Root/TestTable` WHERE Group = 401u OR Group = 402u; )"); auto txControl = NYdb::NQuery::TTxControl::BeginTx().CommitTx(); auto result = session.ExecuteQuery(query, txControl, GetQuerySettings()).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); auto stats = NYdb::TProtoAccessor::GetProto(*result.GetStats()); - Cerr << stats.DebugString() << Endl; - size_t phase = stats.query_phases_size() - 1; - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).deletes().rows(), 1); - UNIT_ASSERT_VALUES_EQUAL(stats.query_phases(phase).table_access(0).deletes().bytes(), 0); + + bool tableDeletesFound = false; + for (const auto& phase : stats.query_phases()) { + for (const auto& tableAccess : phase.table_access()) { + if (tableAccess.name() != testTableName) { + continue; + } + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().rows(), 2); + UNIT_ASSERT_VALUES_EQUAL(tableAccess.deletes().bytes(), 0); + UNIT_ASSERT_C(!tableDeletesFound, "Found two deletes entries"); + tableDeletesFound = true; + } + } + UNIT_ASSERT_C(tableDeletesFound, "No deletes entries found"); } } diff --git a/ydb/core/tx/columnshard/blobs_action/common/statistics.cpp b/ydb/core/tx/columnshard/blobs_action/common/statistics.cpp new file mode 100644 index 000000000000..a9f51011deb3 --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/common/statistics.cpp @@ -0,0 +1,16 @@ +#include "statistics.h" + +namespace NKikimr::NColumnShard { + +void AddTableAccessStatsToTxStats(NKikimrQueryStats::TTxStats& stats, ui64 pathId, ui64 rows, ui64 bytes, + NEvWrite::EModificationType modificationType) { + auto tableStats = stats.AddTableAccessStats(); + tableStats->MutableTableInfo()->SetPathId(pathId); + auto row = modificationType == NEvWrite::EModificationType::Delete ? tableStats->MutableEraseRow() + : tableStats->MutableUpdateRow(); + row->SetCount(rows); + row->SetRows(rows); + row->SetBytes(bytes); +} + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/blobs_action/common/statistics.h b/ydb/core/tx/columnshard/blobs_action/common/statistics.h new file mode 100644 index 000000000000..8269f5e1487c --- /dev/null +++ b/ydb/core/tx/columnshard/blobs_action/common/statistics.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr::NColumnShard { + +void AddTableAccessStatsToTxStats(NKikimrQueryStats::TTxStats& stats, ui64 pathId, ui64 rows, ui64 bytes, + NEvWrite::EModificationType modificationType); + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/blobs_action/common/ya.make b/ydb/core/tx/columnshard/blobs_action/common/ya.make index 0d6ae9574b16..523fdc56614f 100644 --- a/ydb/core/tx/columnshard/blobs_action/common/ya.make +++ b/ydb/core/tx/columnshard/blobs_action/common/ya.make @@ -2,9 +2,11 @@ LIBRARY() SRCS( const.cpp + statistics.cpp ) PEERDIR( + ydb/core/protos ) END() diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp index efd5487f333b..9c4be6243911 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp @@ -1,6 +1,7 @@ #include "tx_blobs_written.h" #include +#include #include #include #include @@ -76,6 +77,8 @@ bool TTxBlobsWritingFinished::DoExecute(TTransactionContext& txc, const TActorCo lock.SetCounter(info.GetInternalGenerationCounter()); writeMeta.GetPathId().SchemeShardLocalPathId.ToProto(lock); auto ev = NEvents::TDataEvents::TEvWriteResult::BuildCompleted(Self->TabletID(), operation->GetLockId(), lock); + AddTableAccessStatsToTxStats(*ev->Record.MutableTxStats(), writeMeta.GetPathId().SchemeShardLocalPathId.GetRawValue(), + writeResult.GetRecordsCount(), writeResult.GetDataSize(), operation->GetModificationType()); Results.emplace_back(std::move(ev), writeMeta.GetSource(), operation->GetCookie()); } } From b6756fa8540ece32e1d5244619c64442c474d564 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 17 Jun 2025 13:55:13 +0300 Subject: [PATCH 22/62] special portions data fetcher (#19666) Co-authored-by: ivanmorozov333 --- .../run/kikimr_services_initializers.cpp | 27 +- .../run/kikimr_services_initializers.h | 10 +- ydb/core/driver_lib/run/run.cpp | 3 +- ydb/core/protos/config.proto | 3 +- ydb/core/protos/console_config.proto | 3 +- .../abstract/storages_manager.cpp | 4 +- .../blobs_action/counters/storage.h | 2 + .../transaction/tx_write_index.cpp | 4 - ydb/core/tx/columnshard/columnshard_impl.cpp | 457 +++++++----------- .../columnshard/columnshard_private_events.h | 8 +- .../tx/columnshard/data_accessor/request.h | 17 + .../tx/columnshard/data_reader/contexts.cpp | 14 + .../tx/columnshard/data_reader/contexts.h | 250 ++++++++++ .../tx/columnshard/data_reader/fetcher.cpp | 55 +++ ydb/core/tx/columnshard/data_reader/fetcher.h | 115 +++++ .../data_reader/fetching_executor.cpp | 28 ++ .../data_reader/fetching_executor.h | 20 + .../data_reader/fetching_steps.cpp | 4 + .../columnshard/data_reader/fetching_steps.h | 262 ++++++++++ ydb/core/tx/columnshard/data_reader/ya.make | 6 + .../engines/changes/abstract/abstract.h | 4 +- .../engines/changes/counters/changes.h | 3 +- ydb/core/tx/columnshard/engines/changes/ttl.h | 11 + .../engines/portions/data_accessor.cpp | 8 +- .../engines/portions/data_accessor.h | 3 +- .../engines/reader/abstract/read_metadata.h | 5 + .../engines/reader/sys_view/chunks/chunks.cpp | 121 +++-- .../engines/reader/sys_view/chunks/chunks.h | 65 +-- .../engines/scheme/versions/versioned_index.h | 6 +- .../conveyor_composite/service/counters.cpp | 1 + .../tx/conveyor_composite/service/counters.h | 1 + .../conveyor_composite/service/workers_pool.h | 1 + .../limiter/grouped_memory/service/counters.h | 13 +- .../limiter/grouped_memory/service/process.h | 6 +- .../tx/limiter/grouped_memory/usage/service.h | 9 +- .../grouped_memory/usage/stage_features.cpp | 16 +- .../grouped_memory/usage/stage_features.h | 4 +- ydb/tests/library/harness/kikimr_config.py | 11 +- 38 files changed, 1152 insertions(+), 428 deletions(-) create mode 100644 ydb/core/tx/columnshard/data_reader/contexts.cpp create mode 100644 ydb/core/tx/columnshard/data_reader/contexts.h create mode 100644 ydb/core/tx/columnshard/data_reader/fetcher.cpp create mode 100644 ydb/core/tx/columnshard/data_reader/fetcher.h create mode 100644 ydb/core/tx/columnshard/data_reader/fetching_executor.cpp create mode 100644 ydb/core/tx/columnshard/data_reader/fetching_executor.h create mode 100644 ydb/core/tx/columnshard/data_reader/fetching_steps.cpp create mode 100644 ydb/core/tx/columnshard/data_reader/fetching_steps.h diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index d981d5122326..9130c57e319c 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -2156,17 +2156,17 @@ void TKqpServiceInitializer::InitializeServices(NActors::TActorSystemSetup* setu } } -TGroupedMemoryLimiterInitializer::TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig) +TScanGroupedMemoryLimiterInitializer::TScanGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } -void TGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { +void TScanGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { NOlap::NGroupedMemoryManager::TConfig serviceConfig; - Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetGroupedMemoryLimiterConfig())); + Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetScanGroupedMemoryLimiterConfig())); if (serviceConfig.IsEnabled()) { TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); - TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_GROUPED_MEMORY_LIMITER"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_SCAN_GROUPED_MEMORY_LIMITER"); auto service = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::CreateService(serviceConfig, countersGroup); @@ -2176,6 +2176,25 @@ void TGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemS } } +TCompGroupedMemoryLimiterInitializer::TCompGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TCompGroupedMemoryLimiterInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + NOlap::NGroupedMemoryManager::TConfig serviceConfig; + Y_ABORT_UNLESS(serviceConfig.DeserializeFromProto(Config.GetCompGroupedMemoryLimiterConfig())); + + if (serviceConfig.IsEnabled()) { + TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> countersGroup = tabletGroup->GetSubgroup("type", "TX_COMP_GROUPED_MEMORY_LIMITER"); + + auto service = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::CreateService(serviceConfig, countersGroup); + + setup->LocalServices.push_back(std::make_pair(NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::MakeServiceId(NodeId), + TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); + } +} + TCompDiskLimiterInitializer::TCompDiskLimiterInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 0d41ea7b8cae..3d1b650577de 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -404,9 +404,15 @@ class TCompDiskLimiterInitializer: public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; -class TGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { +class TScanGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { public: - TGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig); + TScanGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + +class TCompGroupedMemoryLimiterInitializer: public IKikimrServicesInitializer { +public: + TCompGroupedMemoryLimiterInitializer(const TKikimrRunConfig& runConfig); void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 0c1e016f6478..dccfb278f36b 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1700,7 +1700,8 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers } if (serviceMask.EnableGroupedMemoryLimiter) { - sil->AddServiceInitializer(new TGroupedMemoryLimiterInitializer(runConfig)); + sil->AddServiceInitializer(new TScanGroupedMemoryLimiterInitializer(runConfig)); + sil->AddServiceInitializer(new TCompGroupedMemoryLimiterInitializer(runConfig)); } if (serviceMask.EnableCompPriorities) { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index eec56126ac5e..a57b3655b1ad 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -2338,7 +2338,7 @@ message TAppConfig { optional TLimiterConfig CompDiskLimiterConfig = 79; optional TMetadataCacheConfig MetadataCacheConfig = 80; optional TMemoryControllerConfig MemoryControllerConfig = 81; - optional TGroupedMemoryLimiterConfig GroupedMemoryLimiterConfig = 82; + optional TGroupedMemoryLimiterConfig ScanGroupedMemoryLimiterConfig = 82; optional NKikimrReplication.TReplicationDefaults ReplicationConfig = 83; optional TShutdownConfig ShutdownConfig = 84; optional TPrioritiesQueueConfig CompPrioritiesConfig = 85; @@ -2346,6 +2346,7 @@ message TAppConfig { optional NKikimrProto.TDataIntegrityTrailsConfig DataIntegrityTrailsConfig = 87; optional TDataErasureConfig DataErasureConfig = 88; optional THealthCheckConfig HealthCheckConfig = 89; + optional TGroupedMemoryLimiterConfig CompGroupedMemoryLimiterConfig = 92; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 6372aee5f6c7..2943e7e33234 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -140,10 +140,11 @@ message TConfigItem { BackgroundCleaningConfigItem = 77; MetadataCacheConfigItem = 80; MemoryControllerConfigItem = 81; - GroupedMemoryLimiterConfig = 82; + ScanGroupedMemoryLimiterConfig = 82; ReplicationConfigItem = 83; CompPrioritiesConfig = 85; HealthCheckConfigItem = 89; + CompGroupedMemoryLimiterConfig = 92; NamedConfigsItem = 100; ClusterYamlConfigItem = 101; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp index 806ea6b69093..ef37d660c7dc 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp @@ -4,7 +4,7 @@ namespace NKikimr::NOlap { -std::shared_ptr IStoragesManager::GetOperatorOptional(const TString& storageId) const { +std::shared_ptr IStoragesManager::GetOperatorOptional(const TString& storageId) const { AFL_VERIFY(Initialized); AFL_VERIFY(storageId); TReadGuard rg(RWMutex); @@ -16,7 +16,7 @@ std::shared_ptr IStoragesManager::GetOper } } -std::shared_ptr IStoragesManager::GetOperatorVerified(const TString& storageId) const { +std::shared_ptr IStoragesManager::GetOperatorVerified(const TString& storageId) const { auto result = GetOperatorOptional(storageId); AFL_VERIFY(result)("storage_id", storageId); return result; diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.h b/ydb/core/tx/columnshard/blobs_action/counters/storage.h index 94b66e47e7c8..9a94ccfc4a85 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.h @@ -23,12 +23,14 @@ enum class EConsumer { CLEANUP_SHARED_BLOBS, EXPORT, SCAN, + SYS_VIEW_SCAN, GC, WRITING, WRITING_BUFFER, WRITING_OPERATOR, NORMALIZER, STATISTICS, + UNDEFINED, COUNT }; diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index 9719e5854583..a5fa294d5b93 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -15,10 +15,6 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) ACFL_DEBUG("event", "TTxWriteIndex::Execute")("change_type", changes->TypeString())("details", changes->DebugString()); if (Ev->Get()->GetPutStatus() == NKikimrProto::OK) { - AFL_VERIFY(Ev->Get()->IndexInfo->GetLastSchema()->GetSnapshot() <= Self->GetLastTxSnapshot()) - ("schema_last", Ev->Get()->IndexInfo->GetLastSchema()->GetSnapshot().DebugString())( - "planned_last", Self->GetLastTxSnapshot().DebugString()); - TBlobGroupSelector dsGroupSelector(Self->Info()); NOlap::TDbWrapper dbWrap(txc.DB, &dsGroupSelector); AFL_VERIFY(Self->TablesManager.MutablePrimaryIndex().ApplyChangesOnExecute(dbWrap, changes, Self->GetLastTxSnapshot())); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 7437b028f606..179523763194 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -22,6 +22,8 @@ #include "blobs_reader/actor.h" #include "bg_tasks/events/events.h" +#include "data_reader/fetcher.h" +#include "data_reader/contexts.h" #include "data_accessor/manager.h" #include "data_sharing/destination/session/destination.h" #include "data_sharing/source/session/source.h" @@ -444,190 +446,6 @@ void TColumnShard::EnqueueBackgroundActivities(const bool periodic) { SetupGC(); } -class TChangesTask: public NConveyor::ITask { -private: - std::unique_ptr TxEvent; - const TIndexationCounters Counters; - const ui64 TabletId; - const TActorId ParentActorId; - TString ClassId; - NOlap::TSnapshot LastCompletedTx; - -protected: - virtual void DoExecute(const std::shared_ptr& /*taskPtr*/) override { - NActors::TLogContextGuard g(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletId)("parent_id", ParentActorId)); - { - NOlap::TConstructionContext context(*TxEvent->IndexInfo, Counters, LastCompletedTx); - Y_ABORT_UNLESS(TxEvent->IndexChanges->ConstructBlobs(context).Ok()); - if (!TxEvent->IndexChanges->GetWritePortionsCount()) { - TxEvent->SetPutStatus(NKikimrProto::OK); - } - } - TActorContext::AsActorContext().Send(ParentActorId, std::move(TxEvent)); - } - -public: - virtual TString GetTaskClassIdentifier() const override { - return ClassId; - } - - TChangesTask(std::unique_ptr&& txEvent, const TIndexationCounters& counters, const ui64 tabletId, const TActorId parentActorId, NOlap::TSnapshot lastCompletedTx) - : TxEvent(std::move(txEvent)) - , Counters(counters) - , TabletId(tabletId) - , ParentActorId(parentActorId) - , LastCompletedTx(lastCompletedTx) { - Y_ABORT_UNLESS(TxEvent); - Y_ABORT_UNLESS(TxEvent->IndexChanges); - ClassId = "Changes::ConstructBlobs::" + TxEvent->IndexChanges->TypeString(); - } -}; - -class TChangesReadTask: public NOlap::NBlobOperations::NRead::ITask { -private: - using TBase = NOlap::NBlobOperations::NRead::ITask; - const TActorId ParentActorId; - const ui64 TabletId; - std::unique_ptr TxEvent; - TIndexationCounters Counters; - NOlap::TSnapshot LastCompletedTx; - -protected: - virtual void DoOnDataReady(const std::shared_ptr& resourcesGuard) override { - if (!!resourcesGuard) { - AFL_VERIFY(!TxEvent->IndexChanges->ResourcesGuard); - TxEvent->IndexChanges->ResourcesGuard = resourcesGuard; - } else { - AFL_VERIFY(TxEvent->IndexChanges->ResourcesGuard); - } - TxEvent->IndexChanges->Blobs = ExtractBlobsData(); - TxEvent->IndexChanges->SetStage(NOlap::NChanges::EStage::ReadyForConstruct); - std::shared_ptr task = - std::make_shared(std::move(TxEvent), Counters, TabletId, ParentActorId, LastCompletedTx); - NConveyorComposite::TCompServiceOperator::SendTaskToExecute(task); - } - virtual bool DoOnError(const TString& storageId, const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "DoOnError")("storage_id", storageId)("blob_id", range)("status", status.GetErrorMessage())("status_code", status.GetStatus()); - AFL_VERIFY(status.GetStatus() != NKikimrProto::EReplyStatus::NODATA)("blob_id", range)("status", status.GetStatus())("error", status.GetErrorMessage())("type", TxEvent->IndexChanges->TypeString())("task_id", TxEvent->IndexChanges->GetTaskIdentifier())("debug", TxEvent->IndexChanges->DebugString()); - TxEvent->SetPutStatus(NKikimrProto::ERROR); - Counters.ReadErrors->Add(1); - TActorContext::AsActorContext().Send(ParentActorId, std::move(TxEvent)); - return false; - } - -public: - TChangesReadTask(std::unique_ptr&& event, const TActorId parentActorId, const ui64 tabletId, const TIndexationCounters& counters, NOlap::TSnapshot lastCompletedTx) - : TBase(event->IndexChanges->GetReadingActions(), event->IndexChanges->TypeString(), event->IndexChanges->GetTaskIdentifier()) - , ParentActorId(parentActorId) - , TabletId(tabletId) - , TxEvent(std::move(event)) - , Counters(counters) - , LastCompletedTx(lastCompletedTx) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "start_changes")("type", TxEvent->IndexChanges->TypeString())("task_id", TxEvent->IndexChanges->GetTaskIdentifier()); - } -}; - -class TDataAccessorsSubscriberBase: public NOlap::IDataAccessorRequestsSubscriber { -private: - std::shared_ptr ResourcesGuard; - virtual const std::shared_ptr& DoGetAbortionFlag() const override { - return Default>(); - } - - virtual void DoOnRequestsFinished(NOlap::TDataAccessorsResult&& result) override final { - AFL_VERIFY(ResourcesGuard); - DoOnRequestsFinished(std::move(result), std::move(ResourcesGuard)); - } - -protected: - virtual void DoOnRequestsFinished(NOlap::TDataAccessorsResult&& result, std::shared_ptr&& guard) = 0; - -public: - void SetResourcesGuard(const std::shared_ptr& guard) { - AFL_VERIFY(!ResourcesGuard); - AFL_VERIFY(guard); - ResourcesGuard = guard; - } -}; - -class TDataAccessorsSubscriber: public TDataAccessorsSubscriberBase { -protected: - const NActors::TActorId ShardActorId; - std::shared_ptr Changes; - std::shared_ptr VersionedIndex; - std::shared_ptr ResourcesGuard; - - virtual void DoOnRequestsFinishedImpl() = 0; - - virtual void DoOnRequestsFinished(NOlap::TDataAccessorsResult&& result, std::shared_ptr&& guard) override final { - Changes->SetFetchedDataAccessors(std::move(result), NOlap::TDataAccessorsInitializationContext(VersionedIndex)); - Changes->ResourcesGuard = std::move(guard); - DoOnRequestsFinishedImpl(); - } - -public: - void SetResourcesGuard(const std::shared_ptr& guard) { - AFL_VERIFY(!ResourcesGuard); - ResourcesGuard = guard; - } - - std::shared_ptr&& ExtractResourcesGuard() { - AFL_VERIFY(ResourcesGuard); - return std::move(ResourcesGuard); - } - - TDataAccessorsSubscriber(const NActors::TActorId& shardActorId, const std::shared_ptr& changes, - const std::shared_ptr& versionedIndex) - : ShardActorId(shardActorId) - , Changes(changes) - , VersionedIndex(versionedIndex) { - } -}; - -class TDataAccessorsSubscriberWithRead: public TDataAccessorsSubscriber { -private: - using TBase = TDataAccessorsSubscriber; - -protected: - const bool CacheDataAfterWrite = false; - const ui64 ShardTabletId; - TIndexationCounters Counters; - NOlap::TSnapshot SnapshotModification; - const NActors::TActorId ResourceSubscribeActor; - const NOlap::NResourceBroker::NSubscribe::TTaskContext TaskSubscriptionContext; - -public: - TDataAccessorsSubscriberWithRead(const NActors::TActorId& resourceSubscribeActor, const std::shared_ptr& changes, - const std::shared_ptr& versionedIndex, const bool cacheAfterWrite, const NActors::TActorId& shardActorId, - const ui64 shardTabletId, const TIndexationCounters& counters, const NOlap::TSnapshot& snapshotModification, - const NOlap::NResourceBroker::NSubscribe::TTaskContext& taskSubscriptionContext) - : TBase(shardActorId, changes, versionedIndex) - , CacheDataAfterWrite(cacheAfterWrite) - , ShardTabletId(shardTabletId) - , Counters(counters) - , SnapshotModification(snapshotModification) - , ResourceSubscribeActor(resourceSubscribeActor) - , TaskSubscriptionContext(taskSubscriptionContext) - { - } -}; - -class TCompactChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCounter { -private: - using TBase = TChangesReadTask; - -public: - using TBase::TBase; -}; - -class TTTLChangesReadTask: public TChangesReadTask, public TMonitoringObjectsCounter { -private: - using TBase = TChangesReadTask; - -public: - using TBase::TBase; -}; - namespace { class TCompactionAllocated: public NPrioritiesQueue::IRequest { private: @@ -666,53 +484,101 @@ void TColumnShard::SetupCompaction(const std::set& pathIds) { } } -class TAccessorsMemorySubscriber: public NOlap::NResourceBroker::NSubscribe::ITask { +class TCompactionExecutor: public NOlap::NDataFetcher::IFetchCallback { private: - using TBase = NOlap::NResourceBroker::NSubscribe::ITask; - std::shared_ptr ChangeTask; - std::shared_ptr Request; - std::shared_ptr Subscriber; - std::shared_ptr DataAccessorsManager; - - virtual void DoOnAllocationSuccess(const std::shared_ptr& guard) override { - Subscriber->SetResourcesGuard(guard); - Request->RegisterSubscriber(Subscriber); - if (ChangeTask) { - ChangeTask->SetStage(NOlap::NChanges::EStage::AskAccessors); + const ui64 TabletId; + const NActors::TActorId ParentActorId; + std::shared_ptr Changes; + std::shared_ptr VersionedIndex; + std::shared_ptr ResourcesGuard; + const TIndexationCounters Counters; + const NOlap::TSnapshot SnapshotModification; + const bool NeedBlobs = true; + const std::shared_ptr TabletActivity; + + virtual std::optional GetMemoryForUsage() const override { + return Changes->CalcMemoryForUsage(); + } + + virtual bool IsAborted() const override { + return !TabletActivity->Val(); + } + + virtual TString GetClassName() const override { + return Changes->TypeString(); + } + + virtual void OnStageStarting(const NOlap::NDataFetcher::EFetchingStage stage) override { + switch (stage) { + case NOlap::NDataFetcher::EFetchingStage::AskAccessors: + Changes->SetStage(NOlap::NChanges::EStage::AskAccessors); + break; + case NOlap::NDataFetcher::EFetchingStage::AskDataResources: + case NOlap::NDataFetcher::EFetchingStage::AskGeneralResources: + Changes->SetStage(NOlap::NChanges::EStage::AskDataResources); + break; + case NOlap::NDataFetcher::EFetchingStage::AskAccessorResources: + Changes->SetStage(NOlap::NChanges::EStage::AskAccessorResources); + break; + case NOlap::NDataFetcher::EFetchingStage::ReadBlobs: + Changes->SetStage(NOlap::NChanges::EStage::ReadBlobs); + break; + case NOlap::NDataFetcher::EFetchingStage::Finished: + Changes->SetStage(NOlap::NChanges::EStage::ReadyForConstruct); + break; + default: + break; } - DataAccessorsManager->AskData(Request); - } - -public: - TAccessorsMemorySubscriber(const ui64 memory, const TString& externalTaskId, const NOlap::NResourceBroker::NSubscribe::TTaskContext& context, - std::shared_ptr&& request, const std::shared_ptr& subscriber, - const std::shared_ptr& dataAccessorsManager, - const std::shared_ptr& changeTask) - : TBase(0, memory, externalTaskId, context) - , ChangeTask(changeTask) - , Request(std::move(request)) - , Subscriber(subscriber) - , DataAccessorsManager(dataAccessorsManager) { } -}; - -class TCompactionDataAccessorsSubscriber: public TDataAccessorsSubscriberWithRead { -private: - using TBase = TDataAccessorsSubscriberWithRead; -protected: - virtual void DoOnRequestsFinishedImpl() override { - const TString externalTaskId = Changes->GetTaskIdentifier(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "compaction")("external_task_id", externalTaskId); + virtual void DoOnFinished(NOlap::NDataFetcher::TCurrentContext&& context) override { + NActors::TLogContextGuard g( + NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletId)("parent_id", ParentActorId)); + if (NeedBlobs) { + AFL_VERIFY(context.GetResourceGuards().size() == 2); + } else { + AFL_VERIFY(context.GetResourceGuards().size() == 1); + } + if (NeedBlobs) { + Changes->Blobs = context.ExtractBlobs(); + } + Changes->SetFetchedDataAccessors( + NOlap::TDataAccessorsResult(context.ExtractPortionAccessors()), NOlap::TDataAccessorsInitializationContext(VersionedIndex)); - Changes->SetStage(NOlap::NChanges::EStage::ReadBlobs); - auto ev = std::make_unique(VersionedIndex, Changes, CacheDataAfterWrite); - TActorContext::AsActorContext().Register(new NOlap::NBlobOperations::NRead::TActor( - std::make_shared(std::move(ev), ShardActorId, ShardTabletId, Counters, SnapshotModification))); + auto ev = std::make_unique(Changes, false); + Changes->FetchingContext.emplace(std::move(context)); + { + NOlap::TConstructionContext context(*VersionedIndex, Counters, SnapshotModification); + if (NeedBlobs) { + Changes->ConstructBlobs(context).Validate(); + } + if (!Changes->GetWritePortionsCount()) { + ev->SetPutStatus(NKikimrProto::OK); + } + } + TActorContext::AsActorContext().Send(ParentActorId, std::move(ev)); + } + virtual void DoOnError(const TString& errorMessage) override { + auto ev = std::make_unique(Changes, false); + ev->SetPutStatus(NKikimrProto::ERROR); + ev->ErrorMessage = errorMessage; + TActorContext::AsActorContext().Send(ParentActorId, std::move(ev)); } public: - using TBase::TBase; + TCompactionExecutor(const ui64 tabletId, const NActors::TActorId parentActorId, const std::shared_ptr& changes, + const std::shared_ptr& versionedIndex, const TIndexationCounters& counters, + const NOlap::TSnapshot snapshotModification, const std::shared_ptr& tabletActivity, const bool needBlobs = true) + : TabletId(tabletId) + , ParentActorId(parentActorId) + , Changes(changes) + , VersionedIndex(versionedIndex) + , Counters(counters) + , SnapshotModification(snapshotModification) + , NeedBlobs(needBlobs) + , TabletActivity(tabletActivity) + { + } }; void TColumnShard::StartCompaction(const std::shared_ptr& guard) { @@ -731,49 +597,37 @@ void TColumnShard::StartCompaction(const std::shared_ptrGetVersionedIndexReadonlyCopy(); - auto request = compaction.ExtractDataAccessorsRequest(); - const ui64 accessorsMemory = request->PredictAccessorsMemory(TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetLastSchema()) + - indexChanges->CalcMemoryForUsage(); - const auto subscriber = std::make_shared(ResourceSubscribeActor, indexChanges, actualIndexInfo, - Settings.CacheDataAfterCompaction, SelfId(), TabletID(), Counters.GetCompactionCounters(), GetLastCompletedTx(), - CompactTaskSubscription); - compaction.SetStage(NOlap::NChanges::EStage::AskResources); - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription(ResourceSubscribeActor, - std::make_shared(accessorsMemory, indexChanges->GetTaskIdentifier(), CompactTaskSubscription, - std::move(request), - subscriber, DataAccessorsManager.GetObjectPtrVerified(), indexChanges)); + NOlap::NDataFetcher::TRequestInput rInput(compaction.GetSwitchedPortions(), actualIndexInfo, + NOlap::NBlobOperations::EConsumer::GENERAL_COMPACTION, compaction.GetTaskIdentifier()); + auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); + NOlap::NDataFetcher::TPortionsDataFetcher::StartFullPortionsFetching(std::move(rInput), + std::make_shared( + TabletID(), SelfId(), indexChanges, actualIndexInfo, Counters.GetIndexationCounters(), GetLastCompletedTx(), TabletActivityImpl), + env, NConveyorComposite::ESpecialTaskCategory::Compaction); } -class TWriteEvictPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriberWithRead { +class TDataAccessorsSubscriberBase: public NOlap::IDataAccessorRequestsSubscriber { private: - using TBase = TDataAccessorsSubscriberWithRead; - -protected: - virtual void DoOnRequestsFinishedImpl() override { - ACFL_DEBUG("background", "ttl")("need_writes", true); - auto ev = std::make_unique(VersionedIndex, Changes, false); - TActorContext::AsActorContext().Register(new NOlap::NBlobOperations::NRead::TActor( - std::make_shared(std::move(ev), ShardActorId, ShardTabletId, Counters, SnapshotModification))); + std::shared_ptr ResourcesGuard; + virtual const std::shared_ptr& DoGetAbortionFlag() const override { + return Default>(); } -public: - using TBase::TBase; -}; - -class TNoWriteEvictPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriber { -private: - using TBase = TDataAccessorsSubscriber; + virtual void DoOnRequestsFinished(NOlap::TDataAccessorsResult&& result) override final { + AFL_VERIFY(ResourcesGuard); + DoOnRequestsFinished(std::move(result), std::move(ResourcesGuard)); + } protected: - virtual void DoOnRequestsFinishedImpl() override { - ACFL_DEBUG("background", "ttl")("need_writes", false); - auto ev = std::make_unique(VersionedIndex, Changes, false); - ev->SetPutStatus(NKikimrProto::OK); - NActors::TActivationContext::Send(ShardActorId, std::move(ev)); - } + virtual void DoOnRequestsFinished( + NOlap::TDataAccessorsResult&& result, std::shared_ptr&& guard) = 0; public: - using TBase::TBase; + void SetResourcesGuard(const std::shared_ptr& guard) { + AFL_VERIFY(!ResourcesGuard); + AFL_VERIFY(guard); + ResourcesGuard = guard; + } }; class TCSMetadataSubscriber: public TDataAccessorsSubscriberBase, public TObjectCounter { @@ -799,6 +653,36 @@ class TCSMetadataSubscriber: public TDataAccessorsSubscriberBase, public TObject } }; +class TAccessorsMemorySubscriber: public NOlap::NResourceBroker::NSubscribe::ITask { +private: + using TBase = NOlap::NResourceBroker::NSubscribe::ITask; + std::shared_ptr ChangeTask; + std::shared_ptr Request; + std::shared_ptr Subscriber; + std::shared_ptr DataAccessorsManager; + + virtual void DoOnAllocationSuccess(const std::shared_ptr& guard) override { + Subscriber->SetResourcesGuard(guard); + Request->RegisterSubscriber(Subscriber); + if (ChangeTask) { + ChangeTask->SetStage(NOlap::NChanges::EStage::AskAccessors); + } + DataAccessorsManager->AskData(Request); + } + +public: + TAccessorsMemorySubscriber(const ui64 memory, const TString& externalTaskId, const NOlap::NResourceBroker::NSubscribe::TTaskContext& context, + std::shared_ptr&& request, const std::shared_ptr& subscriber, + const std::shared_ptr& dataAccessorsManager, + const std::shared_ptr& changeTask) + : TBase(0, memory, externalTaskId, context) + , ChangeTask(changeTask) + , Request(std::move(request)) + , Subscriber(subscriber) + , DataAccessorsManager(dataAccessorsManager) { + } +}; + void TColumnShard::SetupMetadata() { if (TObjectCounter::ObjectCount()) { return; @@ -834,45 +718,28 @@ bool TColumnShard::SetupTtl() { auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); for (auto&& i : indexChanges) { i->Start(*this); - auto request = i->ExtractDataAccessorsRequest(); - ui64 memoryUsage = 0; - std::shared_ptr subscriber; + NOlap::NDataFetcher::TRequestInput rInput( + i->GetPortionsInfo(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::TTL, i->GetTaskIdentifier()); + auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); if (i->NeedConstruction()) { - subscriber = std::make_shared(ResourceSubscribeActor, i, actualIndexInfo, - Settings.CacheDataAfterCompaction, SelfId(), TabletID(), Counters.GetEvictionCounters(), GetLastCompletedTx(), - TTLTaskSubscription); - memoryUsage = i->CalcMemoryForUsage(); + NOlap::NDataFetcher::TPortionsDataFetcher::StartFullPortionsFetching(std::move(rInput), + std::make_shared( + TabletID(), SelfId(), i, actualIndexInfo, Counters.GetIndexationCounters(), GetLastCompletedTx(), TabletActivityImpl, true), + env, NConveyorComposite::ESpecialTaskCategory::Compaction); } else { - subscriber = std::make_shared(SelfId(), i, actualIndexInfo); + NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching(std::move(rInput), + std::make_shared( + TabletID(), SelfId(), i, actualIndexInfo, Counters.GetIndexationCounters(), GetLastCompletedTx(), TabletActivityImpl, false), + env, NConveyorComposite::ESpecialTaskCategory::Compaction); } - const ui64 accessorsMemory = - request->PredictAccessorsMemory(TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetLastSchema()) + memoryUsage; - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared(accessorsMemory, i->GetTaskIdentifier(), TTLTaskSubscription, - std::move(request), subscriber, DataAccessorsManager.GetObjectPtrVerified(), i)); } return true; } -class TCleanupPortionsDataAccessorsSubscriber: public TDataAccessorsSubscriber { -private: - using TBase = TDataAccessorsSubscriber; - -protected: - virtual void DoOnRequestsFinishedImpl() override { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("background", "cleanup")("changes_info", Changes->DebugString()); - auto ev = std::make_unique(VersionedIndex, Changes, false); - ev->SetPutStatus(NKikimrProto::OK); // No new blobs to write - NActors::TActivationContext::Send(ShardActorId, std::move(ev)); - } - -public: - using TBase::TBase; -}; - void TColumnShard::SetupCleanupPortions() { Counters.GetCSCounters().OnSetupCleanup(); - if (!AppDataVerified().ColumnShardConfig.GetCleanupEnabled() || !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Cleanup)) { + if (!AppDataVerified().ColumnShardConfig.GetCleanupEnabled() || + !NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::Cleanup)) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_cleanup")("reason", "disabled"); return; } @@ -891,14 +758,14 @@ void TColumnShard::SetupCleanupPortions() { } changes->Start(*this); - auto request = changes->ExtractDataAccessorsRequest(); auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); - const ui64 accessorsMemory = request->PredictAccessorsMemory(TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetLastSchema()); - const auto subscriber = std::make_shared(SelfId(), changes, actualIndexInfo); - - NOlap::NResourceBroker::NSubscribe::ITask::StartResourceSubscription( - ResourceSubscribeActor, std::make_shared(accessorsMemory, changes->GetTaskIdentifier(), TTLTaskSubscription, - std::move(request), subscriber, DataAccessorsManager.GetObjectPtrVerified(), changes)); + NOlap::NDataFetcher::TRequestInput rInput( + changes->GetPortionsToDrop(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier()); + auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); + NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching(std::move(rInput), + std::make_shared( + TabletID(), SelfId(), changes, actualIndexInfo, Counters.GetIndexationCounters(), GetLastCompletedTx(), TabletActivityImpl, false), + env, NConveyorComposite::ESpecialTaskCategory::Compaction); } void TColumnShard::SetupCleanupTables() { @@ -921,7 +788,7 @@ void TColumnShard::SetupCleanupTables() { ACFL_DEBUG("background", "cleanup")("changes_info", changes->DebugString()); auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); - auto ev = std::make_unique(actualIndexInfo, changes, false); + auto ev = std::make_unique(changes, false); ev->SetPutStatus(NKikimrProto::OK); // No new blobs to write changes->Start(*this); diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 0b69676d7e18..52b5d8c267ef 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -165,18 +165,16 @@ struct TEvPrivate { /// Common event for Indexing and GranuleCompaction: write index data in TTxWriteIndex transaction. struct TEvWriteIndex: public TEventLocal { - std::shared_ptr IndexInfo; std::shared_ptr IndexChanges; bool GranuleCompaction{ false }; TUsage ResourceUsage; bool CacheData{ false }; TDuration Duration; TBlobPutResult::TPtr PutResult; + TString ErrorMessage; - TEvWriteIndex( - const std::shared_ptr& indexInfo, std::shared_ptr indexChanges, bool cacheData) - : IndexInfo(indexInfo) - , IndexChanges(indexChanges) + TEvWriteIndex(std::shared_ptr indexChanges, bool cacheData) + : IndexChanges(indexChanges) , CacheData(cacheData) { PutResult = std::make_shared(NKikimrProto::UNKNOWN); } diff --git a/ydb/core/tx/columnshard/data_accessor/request.h b/ydb/core/tx/columnshard/data_accessor/request.h index d0eaf8051c73..2b52a5cc4c98 100644 --- a/ydb/core/tx/columnshard/data_accessor/request.h +++ b/ydb/core/tx/columnshard/data_accessor/request.h @@ -15,6 +15,15 @@ class TDataAccessorsResult: private NNonCopyable::TMoveOnly { THashMap PortionsById; public: + TDataAccessorsResult() = default; + + TDataAccessorsResult(std::vector&& portions) { + for (auto&& i : portions) { + const ui64 portionId = i.GetPortionInfo().GetPortionId(); + PortionsById.emplace(portionId, std::move(i)); + } + } + const THashMap& GetPortions() const { return PortionsById; } @@ -43,6 +52,14 @@ class TDataAccessorsResult: private NNonCopyable::TMoveOnly { return it->second; } + TPortionDataAccessor ExtractPortionAccessorVerified(const ui64 portionId) { + auto it = PortionsById.find(portionId); + AFL_VERIFY(it != PortionsById.end()); + auto result = std::move(it->second); + PortionsById.erase(it); + return result; + } + void AddData(THashMap&& accessors) { std::deque v; for (auto&& [portionId, i] : accessors) { diff --git a/ydb/core/tx/columnshard/data_reader/contexts.cpp b/ydb/core/tx/columnshard/data_reader/contexts.cpp new file mode 100644 index 000000000000..f5ffdc7afebc --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/contexts.cpp @@ -0,0 +1,14 @@ +#include "contexts.h" +#include "fetcher.h" + +namespace NKikimr::NOlap::NDataFetcher { + +IFetchingStep::EStepResult IFetchingStep::Execute(const std::shared_ptr& fetchingContext) const { + if (fetchingContext->IsAborted()) { + fetchingContext->OnError("aborted"); + return EStepResult::Error; + } + return DoExecute(fetchingContext); +} + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h new file mode 100644 index 000000000000..203dabf6ef34 --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -0,0 +1,250 @@ +#pragma once +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NDataFetcher { + +enum class EFetchingStage : ui32 { + Created = 0, + AskAccessorResources, + AskDataResources, + AskGeneralResources, + AskAccessors, + ReadBlobs, + Finished, + Error +}; + +class TCurrentContext: TMoveOnly { +private: + std::optional> Accessors; + YDB_READONLY_DEF(std::vector>, ResourceGuards); + std::shared_ptr MemoryProcessGuard; + std::shared_ptr MemoryProcessScopeGuard; + std::shared_ptr MemoryProcessGroupGuard; + static inline TAtomicCounter Counter = 0; + const ui64 MemoryProcessId = Counter.Inc(); + std::optional Blobs; + +public: + ui64 GetMemoryProcessId() const { + return MemoryProcessId; + } + + ui64 GetMemoryScopeId() const { + return MemoryProcessScopeGuard->GetScopeId(); + } + + ui64 GetMemoryGroupId() const { + return MemoryProcessGroupGuard->GetGroupId(); + } + + void SetBlobs(NBlobOperations::NRead::TCompositeReadBlobs&& blobs) { + AFL_VERIFY(!Blobs); + Blobs = std::move(blobs); + } + + NBlobOperations::NRead::TCompositeReadBlobs& MutableBlobs() { + AFL_VERIFY(!!Blobs); + return *Blobs; + } + + NBlobOperations::NRead::TCompositeReadBlobs ExtractBlobs() { + AFL_VERIFY(!!Blobs); + auto result = std::move(*Blobs); + Blobs.reset(); + return result; + } + + void ResetBlobs() { + Blobs.reset(); + } + + TCurrentContext() { + static std::shared_ptr stageFeatures = + NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); + + MemoryProcessGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(MemoryProcessId, { stageFeatures }); + MemoryProcessScopeGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildScopeGuard(MemoryProcessId, 1); + MemoryProcessGroupGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildGroupGuard(MemoryProcessId, 1); + } + + void SetPortionAccessors(std::vector&& acc) { + AFL_VERIFY(!Accessors); + Accessors = std::move(acc); + } + + const std::vector GetPortionAccessors() const { + AFL_VERIFY(Accessors); + return *Accessors; + } + + std::vector ExtractPortionAccessors() { + AFL_VERIFY(Accessors); + auto result = std::move(*Accessors); + Accessors.reset(); + return result; + } + + void RegisterResourcesGuard(std::shared_ptr&& g) { + ResourceGuards.emplace_back(std::move(g)); + } +}; + +class IFetchCallback { +private: + virtual void DoOnFinished(TCurrentContext&& context) = 0; + virtual void DoOnError(const TString& errorMessage) = 0; + bool IsFinished = false; + +public: + virtual ~IFetchCallback() = default; + + virtual std::optional GetMemoryForUsage() const { + return std::nullopt; + } + + virtual bool IsAborted() const = 0; + virtual TString GetClassName() const = 0; + + virtual void OnStageStarting(const EFetchingStage /*stage*/) { + + } + + void OnFinished(TCurrentContext&& context) { + AFL_VERIFY(!IsFinished); + IsFinished = true; + return DoOnFinished(std::move(context)); + } + + void OnError(const TString& errorMessage) { + AFL_VERIFY(!IsFinished); + IsFinished = true; + return DoOnError(errorMessage); + } +}; + +class TEnvironment { +private: + YDB_READONLY_DEF(std::shared_ptr, DataAccessorsManager); + YDB_READONLY_DEF(std::shared_ptr, StoragesManager); + +public: + TEnvironment(const std::shared_ptr& accessorsManager, + const std::shared_ptr& storagesManager) + : DataAccessorsManager(accessorsManager) + , StoragesManager(storagesManager) { + } +}; + +class TPortionsDataFetcher; + +class IFetchingStep { +public: + enum class EStepResult { + Continue, + Detached, + Error + }; + +private: + virtual EStepResult DoExecute(const std::shared_ptr& fetchingContext) const = 0; + +public: + virtual ~IFetchingStep() = default; + + [[nodiscard]] EStepResult Execute(const std::shared_ptr& fetchingContext) const; +}; + +class TScript { +private: + const TString ClassName; + std::vector> Steps; + +public: + const TString& GetClassName() const { + return ClassName; + } + + TScript(std::vector>&& steps, const TString& className) + : ClassName(className) + , Steps(std::move(steps)) { + } + + const std::shared_ptr& GetStep(const ui32 index) const { + AFL_VERIFY(index < Steps.size()); + return Steps[index]; + } + + ui32 GetStepsCount() const { + return Steps.size(); + } +}; + +class TScriptExecution { +private: + std::shared_ptr Script; + ui32 StepIndex = 0; + +public: + TScriptExecution(const std::shared_ptr& script) + : Script(script) { + AFL_VERIFY(Script); + } + + const TString& GetScriptClassName() const { + return Script->GetClassName(); + } + + const std::shared_ptr& GetCurrentStep() const { + return Script->GetStep(StepIndex); + } + + bool IsFinished() const { + return StepIndex == Script->GetStepsCount(); + } + + void Next() { + AFL_VERIFY(!IsFinished()); + ++StepIndex; + } +}; + +class TFullPortionInfo { +private: + YDB_READONLY_DEF(TPortionInfo::TConstPtr, PortionInfo); + YDB_READONLY_DEF(ISnapshotSchema::TPtr, Schema); + +public: + TFullPortionInfo(const TPortionInfo::TConstPtr& portionInfo, const ISnapshotSchema::TPtr& schema) + : PortionInfo(portionInfo) + , Schema(schema) { + } +}; + +class TRequestInput { +private: + YDB_READONLY_DEF(std::vector>, Portions); + YDB_READONLY_DEF(std::shared_ptr, ActualSchema); + YDB_READONLY(NBlobOperations::EConsumer, Consumer, NBlobOperations::EConsumer::UNDEFINED); + YDB_READONLY_DEF(TString, ExternalTaskId); + +public: + TRequestInput(const std::vector& portions, const std::shared_ptr& versions, + const NBlobOperations::EConsumer consumer, const TString& externalTaskId) + : Consumer(consumer) + , ExternalTaskId(externalTaskId) { + AFL_VERIFY(portions.size()); + ActualSchema = versions->GetLastSchema(); + for (auto&& i : portions) { + Portions.emplace_back(std::make_shared(i, versions->GetSchemaVerified(i->GetSchemaVersionVerified()))); + } + } +}; + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.cpp b/ydb/core/tx/columnshard/data_reader/fetcher.cpp new file mode 100644 index 000000000000..c94cb5e6820e --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetcher.cpp @@ -0,0 +1,55 @@ +#include "fetcher.h" +#include "fetching_steps.h" + +namespace NKikimr::NOlap::NDataFetcher { + +void TPortionsDataFetcher::StartColumnsFetching(TRequestInput&& input, std::shared_ptr& entityIds, + std::shared_ptr&& callback, const std::shared_ptr& environment, + const NConveyorComposite::ESpecialTaskCategory conveyorCategory) { + std::shared_ptr script = [&]() { + std::vector> steps; + steps.emplace_back(std::make_shared()); + steps.emplace_back(std::make_shared()); + if (auto mem = callback->GetMemoryForUsage()) { + steps.emplace_back(std::make_shared(entityIds, *mem)); + } else { + steps.emplace_back(std::make_shared(entityIds)); + } + steps.emplace_back(std::make_shared(entityIds)); + return std::make_shared(std::move(steps), "PARTIAL_PORTIONS_FETCHING::" + ::ToString(input.GetConsumer())); + }(); + auto fetcher = std::make_shared(std::move(input), std::move(callback), environment, script, conveyorCategory); + fetcher->Resume(fetcher); +} + +void TPortionsDataFetcher::StartFullPortionsFetching(TRequestInput&& input, std::shared_ptr&& callback, + const std::shared_ptr& environment, const NConveyorComposite::ESpecialTaskCategory conveyorCategory) { + static const std::shared_ptr script = [&]() { + std::vector> steps; + steps.emplace_back(std::make_shared()); + steps.emplace_back(std::make_shared()); + if (auto mem = callback->GetMemoryForUsage()) { + steps.emplace_back(std::make_shared(nullptr, *mem)); + } else { + steps.emplace_back(std::make_shared(nullptr)); + } + steps.emplace_back(std::make_shared(nullptr)); + return std::make_shared(std::move(steps), "FULL_PORTIONS_FETCHING::" + ::ToString(input.GetConsumer())); + }(); + auto fetcher = std::make_shared(std::move(input), std::move(callback), environment, script, conveyorCategory); + fetcher->Resume(fetcher); +} + +void TPortionsDataFetcher::StartAccessorPortionsFetching(TRequestInput&& input, std::shared_ptr&& callback, + const std::shared_ptr& environment, const NConveyorComposite::ESpecialTaskCategory conveyorCategory) { + static const std::shared_ptr script = [&]() { + std::vector> steps; + steps.emplace_back(std::make_shared()); + steps.emplace_back(std::make_shared()); + return std::make_shared(std::move(steps), "ACCESSOR_PORTIONS_FETCHING::" + ::ToString(input.GetConsumer())); + }(); + auto fetcher = std::make_shared(std::move(input), std::move(callback), environment, script, conveyorCategory); + fetcher->Resume(fetcher); +} + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h new file mode 100644 index 000000000000..b0a191fc80ab --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -0,0 +1,115 @@ +#pragma once +#include "contexts.h" +#include "fetching_executor.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NDataFetcher { + +class TPortionsDataFetcher: TNonCopyable { +private: + const TRequestInput Input; + const std::shared_ptr Callback; + TScriptExecution Script; + TCurrentContext CurrentContext; + std::shared_ptr Environment; + const NConveyorComposite::ESpecialTaskCategory ConveyorCategory; + bool IsFinishedFlag = false; + EFetchingStage Stage = EFetchingStage::Created; + +public: + void AskMemoryAllocation(const std::shared_ptr& task) { + NGroupedMemoryManager::TCompMemoryLimiterOperator::SendToAllocation( + CurrentContext.GetMemoryProcessId(), CurrentContext.GetMemoryScopeId(), CurrentContext.GetMemoryGroupId(), { task }, 0); + } + + TPortionsDataFetcher(TRequestInput&& input, std::shared_ptr&& callback, const std::shared_ptr& environment, + const std::shared_ptr& script, const NConveyorComposite::ESpecialTaskCategory conveyorCategory) + : Input(std::move(input)) + , Callback(std::move(callback)) + , Script(script) + , Environment(environment) + , ConveyorCategory(conveyorCategory) { + AFL_VERIFY(Environment); + AFL_VERIFY(Callback); + } + + ~TPortionsDataFetcher() { + AFL_VERIFY(IsFinishedFlag || Stage == EFetchingStage::Created || Callback->IsAborted())("stage", Stage)("class_name", Callback->GetClassName()); + } + + static void StartAccessorPortionsFetching(TRequestInput&& input, std::shared_ptr&& callback, + const std::shared_ptr& environment, const NConveyorComposite::ESpecialTaskCategory conveyorCategory); + + static void StartFullPortionsFetching(TRequestInput&& input, std::shared_ptr&& callback, + const std::shared_ptr& environment, const NConveyorComposite::ESpecialTaskCategory conveyorCategory); + + static void StartColumnsFetching(TRequestInput&& input, std::shared_ptr& entityIds, + std::shared_ptr&& callback, const std::shared_ptr& environment, + const NConveyorComposite::ESpecialTaskCategory conveyorCategory); + + TScriptExecution& MutableScript() { + return Script; + } + + const TRequestInput& GetInput() const { + return Input; + } + + const TEnvironment& GetEnvironment() const { + return *Environment; + } + + TCurrentContext& MutableCurrentContext() { + return CurrentContext; + } + + const TCurrentContext& GetCurrentContext() const { + return CurrentContext; + } + + void SetStage(const EFetchingStage stage) { + Callback->OnStageStarting(stage); + Stage = stage; + } + + void OnError(const TString& errMessage) { + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("event", "on_error")("consumer", Input.GetConsumer())( + "task_id", Input.GetExternalTaskId())("script", Script.GetScriptClassName()); + AFL_VERIFY(!IsFinishedFlag); + IsFinishedFlag = true; + SetStage(EFetchingStage::Error); + Callback->OnError(errMessage); + } + + void OnFinished() { + NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("event", "on_finished")("consumer", Input.GetConsumer())( + "task_id", Input.GetExternalTaskId())("script", Script.GetScriptClassName()); + AFL_VERIFY(!IsFinishedFlag); + IsFinishedFlag = true; + SetStage(EFetchingStage::Finished); + Callback->OnFinished(std::move(CurrentContext)); + } + + bool IsAborted() const { + return Callback->IsAborted(); + } + + bool Resume(std::shared_ptr& selfPtr) { + if (IsFinishedFlag) { + return false; + } + NConveyorComposite::TServiceOperator::SendTaskToExecute(std::make_shared(selfPtr), ConveyorCategory, 0); + return true; + } +}; + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetching_executor.cpp b/ydb/core/tx/columnshard/data_reader/fetching_executor.cpp new file mode 100644 index 000000000000..e619be42f0cf --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetching_executor.cpp @@ -0,0 +1,28 @@ +#include "fetching_executor.h" +#include "fetcher.h" + +namespace NKikimr::NOlap::NDataFetcher { + +void TFetchingExecutor::DoExecute(const std::shared_ptr& /*taskPtr*/) { + NActors::TLogContextGuard lGuard = + NActors::TLogContextBuilder::Build()("event", "on_execution")("consumer", Fetcher->GetInput().GetConsumer())( + "task_id", Fetcher->GetInput().GetExternalTaskId())("script", Fetcher->MutableScript().GetScriptClassName()); + while (!Fetcher->MutableScript().IsFinished()) { + switch (Fetcher->MutableScript().GetCurrentStep()->Execute(Fetcher)) { + case IFetchingStep::EStepResult::Continue: + Fetcher->MutableScript().Next(); + break; + case IFetchingStep::EStepResult::Detached: + return; + case IFetchingStep::EStepResult::Error: + return; + } + } + Fetcher->OnFinished(); +} + +TString TFetchingExecutor::GetTaskClassIdentifier() const { + return Fetcher->MutableScript().GetScriptClassName(); +} + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetching_executor.h b/ydb/core/tx/columnshard/data_reader/fetching_executor.h new file mode 100644 index 000000000000..38d0cffce089 --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetching_executor.h @@ -0,0 +1,20 @@ +#pragma once +#include +namespace NKikimr::NOlap::NDataFetcher { + +class TPortionsDataFetcher; + +class TFetchingExecutor: public NConveyorComposite::ITask { +private: + std::shared_ptr Fetcher; + virtual void DoExecute(const std::shared_ptr& taskPtr) override; + +public: + virtual TString GetTaskClassIdentifier() const override; + + TFetchingExecutor(const std::shared_ptr& fetcher) + : Fetcher(fetcher) { + } +}; + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetching_steps.cpp b/ydb/core/tx/columnshard/data_reader/fetching_steps.cpp new file mode 100644 index 000000000000..6e882fc5c8f4 --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetching_steps.cpp @@ -0,0 +1,4 @@ +#include "fetching_steps.h" + +namespace NKikimr::NOlap::NDataFetcher { +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetching_steps.h b/ydb/core/tx/columnshard/data_reader/fetching_steps.h new file mode 100644 index 000000000000..f78fc962371d --- /dev/null +++ b/ydb/core/tx/columnshard/data_reader/fetching_steps.h @@ -0,0 +1,262 @@ +#pragma once +#include "fetcher.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NDataFetcher { + +class TPortionsDataFetcher; + +class TAskAccessorResourcesStep: public IFetchingStep { +private: + class TSubscriber: public NGroupedMemoryManager::IAllocation { + private: + using TBase = NGroupedMemoryManager::IAllocation; + std::shared_ptr FetchingContext; + + virtual void DoOnAllocationImpossible(const TString& errorMessage) override { + FetchingContext->OnError(errorMessage); + } + virtual bool DoOnAllocated(std::shared_ptr&& guard, + const std::shared_ptr& /*allocation*/) override { + FetchingContext->MutableCurrentContext().RegisterResourcesGuard(std::move(guard)); + FetchingContext->MutableScript().Next(); + return FetchingContext->Resume(FetchingContext); + } + + public: + TSubscriber(const ui64 memory, const std::shared_ptr& fetchingContext) + : TBase(memory) + , FetchingContext(fetchingContext) { + } + }; + + virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { + fetchingContext->SetStage(EFetchingStage::AskAccessorResources); + auto request = std::make_shared(::ToString(fetchingContext->GetInput().GetConsumer())); + for (auto&& i : fetchingContext->GetInput().GetPortions()) { + request->AddPortion(i->GetPortionInfo()); + } + const ui64 memory = request->PredictAccessorsMemory(fetchingContext->GetInput().GetActualSchema()); + if (!memory) { + return IFetchingStep::EStepResult::Continue; + } + fetchingContext->AskMemoryAllocation(std::make_shared(memory, fetchingContext)); + return IFetchingStep::EStepResult::Detached; + } + +public: +}; + +class TAskAccessorsStep: public IFetchingStep { +private: + class TSubscriber: public IDataAccessorRequestsSubscriber { + private: + std::shared_ptr Fetcher; + + protected: + virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { + if (result.HasErrors()) { + Fetcher->OnError("cannot fetch accessors"); + } else { + AFL_VERIFY(result.GetPortions().size() == Fetcher->GetInput().GetPortions().size()); + std::vector accessors; + for (auto&& i : Fetcher->GetInput().GetPortions()) { + accessors.emplace_back(result.ExtractPortionAccessorVerified(i->GetPortionInfo()->GetPortionId())); + } + Fetcher->MutableCurrentContext().SetPortionAccessors(std::move(accessors)); + Fetcher->MutableScript().Next(); + Fetcher->Resume(Fetcher); + } + } + virtual const std::shared_ptr& DoGetAbortionFlag() const override { + return Default>(); + } + + public: + TSubscriber(const std::shared_ptr& fetcher) + : Fetcher(fetcher) { + } + }; + + virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { + fetchingContext->SetStage(EFetchingStage::AskAccessors); + std::shared_ptr request = + std::make_shared(::ToString(fetchingContext->GetInput().GetConsumer())); + request->RegisterSubscriber(std::make_shared(fetchingContext)); + for (auto&& i : fetchingContext->GetInput().GetPortions()) { + request->AddPortion(i->GetPortionInfo()); + } + fetchingContext->GetEnvironment().GetDataAccessorsManager()->AskData(request); + return IFetchingStep::EStepResult::Detached; + } + +public: +}; + +class TAskDataResourceStep: public IFetchingStep { +private: + std::shared_ptr ColumnIds; + std::optional MemoryUsage; + + class TSubscriber: public NGroupedMemoryManager::IAllocation { + private: + using TBase = NGroupedMemoryManager::IAllocation; + std::shared_ptr FetchingContext; + + virtual void DoOnAllocationImpossible(const TString& errorMessage) override { + FetchingContext->OnError(errorMessage); + } + virtual bool DoOnAllocated(std::shared_ptr&& guard, + const std::shared_ptr& /*allocation*/) override { + FetchingContext->MutableCurrentContext().RegisterResourcesGuard(std::move(guard)); + FetchingContext->MutableScript().Next(); + return FetchingContext->Resume(FetchingContext); + } + + public: + TSubscriber(const ui64 memory, const std::shared_ptr& fetchingContext) + : TBase(memory) + , FetchingContext(fetchingContext) { + } + }; + + virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { + fetchingContext->SetStage(EFetchingStage::AskDataResources); + const std::vector& accessors = fetchingContext->GetCurrentContext().GetPortionAccessors(); + ui64 memory = 0; + for (auto&& i : accessors) { + if (ColumnIds) { + memory += i.GetColumnBlobBytes(ColumnIds->GetColumnIds()); + } else { + memory += i.GetPortionInfo().GetTotalBlobBytes(); + } + } + if (!memory) { + return IFetchingStep::EStepResult::Continue; + } + fetchingContext->AskMemoryAllocation(std::make_shared(memory, fetchingContext)); + return IFetchingStep::EStepResult::Detached; + } + +public: + TAskDataResourceStep(const std::shared_ptr& columnIds) + : ColumnIds(columnIds) + { + } +}; + +class TAskGeneralResourceStep: public IFetchingStep { +private: + std::shared_ptr ColumnIds; + const ui64 MemoryUsage; + + class TSubscriber: public NGroupedMemoryManager::IAllocation { + private: + using TBase = NGroupedMemoryManager::IAllocation; + std::shared_ptr FetchingContext; + + virtual void DoOnAllocationImpossible(const TString& errorMessage) override { + FetchingContext->OnError(errorMessage); + } + virtual bool DoOnAllocated(std::shared_ptr&& guard, + const std::shared_ptr& /*allocation*/) override { + FetchingContext->MutableCurrentContext().RegisterResourcesGuard(std::move(guard)); + FetchingContext->MutableScript().Next(); + return FetchingContext->Resume(FetchingContext); + } + + public: + TSubscriber(const ui64 memory, const std::shared_ptr& fetchingContext) + : TBase(memory) + , FetchingContext(fetchingContext) { + } + }; + + virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { + fetchingContext->SetStage(EFetchingStage::AskGeneralResources); + fetchingContext->AskMemoryAllocation(std::make_shared(MemoryUsage, fetchingContext)); + return IFetchingStep::EStepResult::Detached; + } + +public: + TAskGeneralResourceStep(const std::shared_ptr& columnIds, const ui64 memoryUsage) + : ColumnIds(columnIds) + , MemoryUsage(memoryUsage) + { + } +}; + +class TAskDataStep: public IFetchingStep { +private: + std::shared_ptr ColumnIds; + + class TSubscriber: public NOlap::NBlobOperations::NRead::ITask { + private: + using TBase = NOlap::NBlobOperations::NRead::ITask; + std::shared_ptr FetchingContext; + + protected: + virtual void DoOnDataReady(const std::shared_ptr& /*resourcesGuard*/) override { + FetchingContext->MutableCurrentContext().SetBlobs(ExtractBlobsData()); + FetchingContext->MutableScript().Next(); + FetchingContext->Resume(FetchingContext); + } + virtual bool DoOnError( + const TString& storageId, const NOlap::TBlobRange& range, const NOlap::IBlobsReadingAction::TErrorStatus& status) override { + FetchingContext->OnError("cannot read blob range: " + storageId + "::" + range.ToString() + "::" + status.GetErrorMessage()); + return false; + } + + public: + TSubscriber( + const std::shared_ptr& fetchingContext, std::vector>&& readActions) + : TBase(readActions, ::ToString(fetchingContext->GetInput().GetConsumer()), fetchingContext->GetInput().GetExternalTaskId()) + , FetchingContext(fetchingContext) { + } + }; + + virtual EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { + fetchingContext->SetStage(EFetchingStage::ReadBlobs); + const std::vector& accessors = fetchingContext->GetCurrentContext().GetPortionAccessors(); + THashMap> ranges; + for (ui32 idx = 0; idx < accessors.size(); ++idx) { + if (ColumnIds) { + accessors[idx].FillBlobRangesByStorage(ranges, fetchingContext->GetInput().GetPortions()[idx]->GetSchema()->GetIndexInfo()); + } else { + accessors[idx].FillBlobRangesByStorage( + ranges, fetchingContext->GetInput().GetPortions()[idx]->GetSchema()->GetIndexInfo(), &ColumnIds->GetColumnIds()); + } + } + std::vector> readActions; + for (auto&& i : ranges) { + auto blobsOperator = fetchingContext->GetEnvironment().GetStoragesManager()->GetOperatorVerified(i.first); + auto readBlobs = blobsOperator->StartReadingAction(fetchingContext->GetInput().GetConsumer()); + for (auto&& br : i.second) { + readBlobs->AddRange(br); + } + readActions.emplace_back(readBlobs); + } + if (readActions.size()) { + TActorContext::AsActorContext().Register( + new NOlap::NBlobOperations::NRead::TActor(std::make_shared(fetchingContext, std::move(readActions)))); + return IFetchingStep::EStepResult::Detached; + } else { + return IFetchingStep::EStepResult::Continue; + } + } + +public: + TAskDataStep(const std::shared_ptr& columnIds) + : ColumnIds(columnIds) { + } +}; + +} // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/ya.make b/ydb/core/tx/columnshard/data_reader/ya.make index c693942fb93c..3264fd73d281 100644 --- a/ydb/core/tx/columnshard/data_reader/ya.make +++ b/ydb/core/tx/columnshard/data_reader/ya.make @@ -2,6 +2,10 @@ LIBRARY() SRCS( actor.cpp + fetcher.cpp + fetching_executor.cpp + fetching_steps.cpp + contexts.cpp ) PEERDIR( @@ -12,4 +16,6 @@ PEERDIR( ydb/core/kqp/compute_actor ) +GENERATE_ENUM_SERIALIZATION(contexts.h) + END() diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index 6745403a5ac6..b478eda7c637 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include #include @@ -379,7 +381,7 @@ class TColumnEngineChanges: public TMoveOnly { void Compile(TFinalizationContext& context) noexcept; NBlobOperations::NRead::TCompositeReadBlobs Blobs; - std::shared_ptr ResourcesGuard; + std::optional FetchingContext; std::vector> GetReadingActions() const { auto result = BlobsAction.GetReadingActions(); diff --git a/ydb/core/tx/columnshard/engines/changes/counters/changes.h b/ydb/core/tx/columnshard/engines/changes/counters/changes.h index de0d1a76009f..6f1ab42ec698 100644 --- a/ydb/core/tx/columnshard/engines/changes/counters/changes.h +++ b/ydb/core/tx/columnshard/engines/changes/counters/changes.h @@ -13,8 +13,9 @@ namespace NKikimr::NOlap::NChanges { enum class EStage : ui32 { Created = 0, Started, - AskResources, + AskAccessorResources, AskAccessors, + AskDataResources, ReadBlobs, ReadyForConstruct, Constructed, diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.h b/ydb/core/tx/columnshard/engines/changes/ttl.h index 5c4b515cf8db..2406f866f717 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.h +++ b/ydb/core/tx/columnshard/engines/changes/ttl.h @@ -112,6 +112,17 @@ class TTTLColumnEngineChanges: public TChangesWithAppend, public NColumnShard::T PortionsToAccess->AddPortion(info); } + std::vector GetPortionsInfo() const { + std::vector result; + for (auto& p : PortionsToEvict) { + result.emplace_back(p.GetPortionInfo()); + } + for (auto& p : GetPortionsToRemove().GetPortionsToRemove()) { + result.emplace_back(p.second); + } + return result; + } + static TString StaticTypeName() { return "CS::TTL"; } diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp index c58ae15bed8b..53323737b126 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp @@ -114,13 +114,19 @@ void TPortionDataAccessor::FillBlobRangesByStorage(THashMapGetIndexInfo()); } -void TPortionDataAccessor::FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo) const { +void TPortionDataAccessor::FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo, const std::set* entityIds) const { for (auto&& i : GetRecordsVerified()) { + if (entityIds && !entityIds->contains(i.GetColumnId())) { + continue; + } const TString& storageId = PortionInfo->GetColumnStorageId(i.GetColumnId(), indexInfo); AFL_VERIFY(result[storageId].emplace(PortionInfo->RestoreBlobRange(i.GetBlobRange())).second)( "blob_id", PortionInfo->RestoreBlobRange(i.GetBlobRange()).ToString()); } for (auto&& i : GetIndexesVerified()) { + if (entityIds && !entityIds->contains(i.GetIndexId())) { + continue; + } const TString& storageId = PortionInfo->GetIndexStorageId(i.GetIndexId(), indexInfo); if (auto bRange = i.GetBlobRangeOptional()) { AFL_VERIFY(result[storageId].emplace(PortionInfo->RestoreBlobRange(*bRange)).second)( diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index 48a34dfd78c5..cdb9723dd721 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -424,7 +424,8 @@ class TPortionDataAccessor { ui64 GetIndexRawBytes(const std::set& entityIds, const bool validation = true) const; ui64 GetIndexRawBytes(const bool validation = true) const; - void FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo) const; + void FillBlobRangesByStorage( + THashMap>& result, const TIndexInfo& indexInfo, const std::set* entityIds = nullptr) const; void FillBlobRangesByStorage(THashMap>& result, const TVersionedIndex& index) const; void FillBlobRangesByStorage(THashMap>>& result, const TIndexInfo& indexInfo, const THashSet& entityIds) const; void FillBlobRangesByStorage( diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h index acd53c064bbb..aa046bbf12b7 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_metadata.h @@ -121,6 +121,11 @@ class TReadMetadataBase { return *IndexVersionsPointer; } + const std::shared_ptr& GetIndexVersionsPtr() const { + AFL_VERIFY(IndexVersionsPointer); + return IndexVersionsPointer; + } + const std::optional& GetRequestShardingInfo() const { return RequestShardingInfo; } diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 961397d9d363..b6499e30ad34 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -1,12 +1,14 @@ #include "chunks.h" #include +#include +#include #include namespace NKikimr::NOlap::NReader::NSysView::NChunks { -void TStatsIterator::AppendStats( - const std::vector>& builders, const NColumnShard::TSchemeShardLocalPathId schemshardLocalPathId, const TPortionDataAccessor& portionPtr) const { +void TStatsIterator::AppendStats(const std::vector>& builders, + const NColumnShard::TSchemeShardLocalPathId schemshardLocalPathId, const TPortionDataAccessor& portionPtr) const { const TPortionInfo& portion = portionPtr.GetPortionInfo(); auto portionSchema = ReadMetadata->GetLoadSchemaVerified(portion); auto it = PortionType.find(portion.GetProduced()); @@ -169,22 +171,88 @@ ui32 TStatsIterator::PredictRecordsCount(const NAbstract::TGranuleMetaView& gran return recordsCount; } +namespace { + +class TApplyResult: public IApplyAction { +private: + using TBase = IDataTasksProcessor::ITask; + YDB_READONLY_DEF(std::vector, Accessors); + +public: + TApplyResult(const std::vector& accessors) + : Accessors(accessors) { + } + + virtual bool DoApply(IDataReader& /*indexedDataRead*/) const override { + AFL_VERIFY(false); + return false; + } +}; + +class TFetchingExecutor: public NOlap::NDataFetcher::IFetchCallback { +private: + const NActors::TActorId ParentActorId; + NColumnShard::TCounterGuard WaitingCountersGuard; + std::shared_ptr AbortionFlag; + + virtual bool IsAborted() const override { + return AbortionFlag->Val(); + } + + virtual TString GetClassName() const override { + return "SYS_VIEW"; + } + + virtual void DoOnFinished(NOlap::NDataFetcher::TCurrentContext&& context) override { + NActors::TActivationContext::AsActorContext().Send( + ParentActorId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult( + std::make_shared(context.ExtractPortionAccessors()), std::move(WaitingCountersGuard))); + } + virtual void DoOnError(const TString& errorMessage) override { + NActors::TActivationContext::AsActorContext().Send( + ParentActorId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult( + TConclusionStatus::Fail("cannot fetch accessors: " + errorMessage), std::move(WaitingCountersGuard))); + } + +public: + TFetchingExecutor(const std::shared_ptr& context) + : ParentActorId(context->GetScanActorId()) + , WaitingCountersGuard(context->GetCounters().GetFetcherAcessorsGuard()) + , AbortionFlag(context->GetAbortionFlag()) + { + } +}; + +} // namespace + TConclusionStatus TStatsIterator::Start() { - ProcessGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(ReadMetadata->GetTxId(), {}); - ScopeGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildScopeGuard(ReadMetadata->GetTxId(), 1); - const ui32 columnsCount = ReadMetadata->GetKeyYqlSchema().size(); + std::vector portions; + std::shared_ptr actualIndexInfo; + auto env = std::make_shared(Context->GetDataAccessorsManager(), Context->GetStoragesManager()); for (auto&& i : IndexGranules) { - GroupGuards.emplace_back(NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard(ReadMetadata->GetTxId(), 1)); for (auto&& p : i.GetPortions()) { - std::shared_ptr request = std::make_shared("SYS_VIEW::CHUNKS"); - request->AddPortion(p); - auto allocation = std::make_shared(request, p->PredictMetadataMemorySize(columnsCount), Context); - request->RegisterSubscriber(allocation); - - NGroupedMemoryManager::TScanMemoryLimiterOperator::SendToAllocation( - ProcessGuard->GetProcessId(), ScopeGuard->GetScopeId(), GroupGuards.back()->GetGroupId(), { allocation }, std::nullopt); + portions.emplace_back(p); + if (portions.size() == 100) { + if (!actualIndexInfo) { + actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); + } + NOlap::NDataFetcher::TRequestInput rInput( + std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId())); + NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( + std::move(rInput), std::make_shared(Context), env, NConveyorComposite::ESpecialTaskCategory::Scan); + portions.clear(); + } } } + if (portions.size()) { + if (!actualIndexInfo) { + actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); + } + NOlap::NDataFetcher::TRequestInput rInput( + std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId())); + NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( + std::move(rInput), std::make_shared(Context), env, NConveyorComposite::ESpecialTaskCategory::Scan); + } return TConclusionStatus::Success(); } @@ -204,23 +272,16 @@ bool TStatsIterator::IsReadyForBatch() const { return false; } -TStatsIterator::TFetchingAccessorAllocation::TFetchingAccessorAllocation( - const std::shared_ptr& request, const ui64 mem, const std::shared_ptr& context) - : TBase(mem) - , AccessorsManager(context->GetDataAccessorsManager()) - , Request(request) - , WaitingCountersGuard(context->GetCounters().GetFetcherAcessorsGuard()) - , OwnerId(context->GetScanActorId()) - , Context(context) { -} - -void TStatsIterator::TFetchingAccessorAllocation::DoOnAllocationImpossible(const TString& errorMessage) { - Request = nullptr; - Context->AbortWithError("cannot allocate memory for take accessors info: " + errorMessage); -} - -const std::shared_ptr& TStatsIterator::TFetchingAccessorAllocation::DoGetAbortionFlag() const { - return Context->GetAbortionFlag(); +void TStatsIterator::Apply(const std::shared_ptr& task) { + if (IndexGranules.empty()) { + return; + } + auto result = std::dynamic_pointer_cast(task); + AFL_VERIFY(result); + for (auto&& i : result->GetAccessors()) { + FetchedAccessors.emplace(i.GetPortionInfo().GetPortionId(), i); + } + } } // namespace NKikimr::NOlap::NReader::NSysView::NChunks diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h index 828cabbe8d64..e5f893024ef6 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.h @@ -58,9 +58,6 @@ class TStatsIterator: public NAbstract::TStatsIterator ColumnNamesById; mutable THashMap PortionType; mutable THashMap> EntityStorageNames; - std::shared_ptr ProcessGuard; - std::shared_ptr ScopeGuard; - std::vector> GroupGuards; using TBase = NAbstract::TStatsIterator; @@ -70,67 +67,7 @@ class TStatsIterator: public NAbstract::TStatsIterator>& builders, const NColumnShard::TSchemeShardLocalPathId schemshardLocalPathId, const TPortionDataAccessor& portion) const; - class TApplyResult: public IApplyAction { - private: - using TBase = IDataTasksProcessor::ITask; - YDB_READONLY_DEF(std::vector, Accessors); - - public: - TApplyResult(const std::vector& accessors) - : Accessors(accessors) { - } - - virtual bool DoApply(IDataReader& /*indexedDataRead*/) const override { - AFL_VERIFY(false); - return false; - } - }; - - class TFetchingAccessorAllocation: public NGroupedMemoryManager::IAllocation, public IDataAccessorRequestsSubscriber { - private: - using TBase = NGroupedMemoryManager::IAllocation; - std::shared_ptr Guard; - std::shared_ptr AccessorsManager; - std::shared_ptr Request; - NColumnShard::TCounterGuard WaitingCountersGuard; - const NActors::TActorId OwnerId; - const std::shared_ptr Context; - - virtual const std::shared_ptr& DoGetAbortionFlag() const override; - virtual bool DoOnAllocated(std::shared_ptr&& guard, - const std::shared_ptr& /*selfPtr*/) override { - Guard = std::move(guard); - AccessorsManager->AskData(std::move(Request)); - return true; - } - virtual void DoOnAllocationImpossible(const TString& errorMessage) override; - - virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { - if (result.HasErrors()) { - NActors::TActivationContext::AsActorContext().Send( - OwnerId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult( - TConclusionStatus::Fail("cannot fetch accessors"), std::move(WaitingCountersGuard))); - } else { - AFL_VERIFY(result.GetPortions().size() == 1)("count", result.GetPortions().size()); - NActors::TActivationContext::AsActorContext().Send( - OwnerId, new NColumnShard::TEvPrivate::TEvTaskProcessedResult( - std::make_shared(result.ExtractPortionsVector()), std::move(WaitingCountersGuard))); - } - } - - public: - TFetchingAccessorAllocation(const std::shared_ptr& request, const ui64 mem, const std::shared_ptr& context); - }; - - virtual void Apply(const std::shared_ptr& task) override { - if (IndexGranules.empty()) { - return; - } - auto result = std::dynamic_pointer_cast(task); - AFL_VERIFY(result); - AFL_VERIFY(result->GetAccessors().size() == 1); - FetchedAccessors.emplace(result->GetAccessors().front().GetPortionInfo().GetPortionId(), result->GetAccessors().front()); - } + virtual void Apply(const std::shared_ptr& task) override; virtual TConclusionStatus Start() override; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h index 259536ffe647..294c5ae99eb9 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h @@ -1,9 +1,9 @@ #pragma once #include "abstract_scheme.h" +#include #include #include -#include #include namespace NKikimr::NOlap { @@ -18,7 +18,8 @@ class TGranuleShardingInfo { YDB_READONLY_DEF(TInternalPathId, PathId); public: - TGranuleShardingInfo(const NSharding::TGranuleShardingLogicContainer& shardingInfo, const TSnapshot& sinceSnapshot, const ui64 version, const TInternalPathId pathId) + TGranuleShardingInfo(const NSharding::TGranuleShardingLogicContainer& shardingInfo, const TSnapshot& sinceSnapshot, const ui64 version, + const TInternalPathId pathId) : ShardingInfo(shardingInfo) , SinceSnapshot(sinceSnapshot) , SnapshotVersion(version) @@ -28,6 +29,7 @@ class TGranuleShardingInfo { }; class TVersionedIndex { +private: THashMap> ShardingInfo; std::map Snapshots; std::shared_ptr PrimaryKey; diff --git a/ydb/core/tx/conveyor_composite/service/counters.cpp b/ydb/core/tx/conveyor_composite/service/counters.cpp index 386c64427507..7db41fcc13cb 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.cpp +++ b/ydb/core/tx/conveyor_composite/service/counters.cpp @@ -27,6 +27,7 @@ TWPCategorySignals::TWPCategorySignals(NColumnShard::TCommonCountersOwner& base, , Category(cat) , WaitingHistogram(TBase::GetHistogram("Waiting/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) , TaskExecuteHistogram(TBase::GetHistogram("TaskExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , ValueWeight(TBase::GetValue("Weight")) , ExecuteDuration(TBase::GetDeriviative("Execute/Duration/Us")) { } diff --git a/ydb/core/tx/conveyor_composite/service/counters.h b/ydb/core/tx/conveyor_composite/service/counters.h index cc85191907db..5c21e77af2ab 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.h +++ b/ydb/core/tx/conveyor_composite/service/counters.h @@ -37,6 +37,7 @@ class TWPCategorySignals: public NColumnShard::TCommonCountersOwner { public: const ::NMonitoring::THistogramPtr WaitingHistogram; const ::NMonitoring::THistogramPtr TaskExecuteHistogram; + const ::NMonitoring::TDynamicCounters::TCounterPtr ValueWeight; const ::NMonitoring::TDynamicCounters::TCounterPtr ExecuteDuration; diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.h b/ydb/core/tx/conveyor_composite/service/workers_pool.h index 3fb08814e0e9..8ed7bf32ed35 100644 --- a/ydb/core/tx/conveyor_composite/service/workers_pool.h +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.h @@ -20,6 +20,7 @@ class TWeightedCategory { , Counters(counters) , Weight(weight) { + counters->ValueWeight->Set(weight); AFL_VERIFY(Counters); AFL_VERIFY(cat); AFL_VERIFY(Weight); diff --git a/ydb/core/tx/limiter/grouped_memory/service/counters.h b/ydb/core/tx/limiter/grouped_memory/service/counters.h index 2e75604b5792..64a52520f942 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/counters.h +++ b/ydb/core/tx/limiter/grouped_memory/service/counters.h @@ -8,18 +8,27 @@ class TStageCounters: public NColumnShard::TCommonCountersOwner { using TBase = NColumnShard::TCommonCountersOwner; NMonitoring::TDynamicCounters::TCounterPtr AllocatedBytes; NMonitoring::TDynamicCounters::TCounterPtr AllocatedChunks; + NMonitoring::TDynamicCounters::TCounterPtr Allocations; + NMonitoring::TDynamicCounters::TCounterPtr Free; NMonitoring::TDynamicCounters::TCounterPtr WaitingBytes; NMonitoring::TDynamicCounters::TCounterPtr WaitingChunks; NMonitoring::TDynamicCounters::TCounterPtr AllocationFailCount; public: + const NMonitoring::TDynamicCounters::TCounterPtr ValueHardLimit; + const NMonitoring::TDynamicCounters::TCounterPtr ValueSoftLimit; + TStageCounters(const TCommonCountersOwner& owner, const TString& name) : TBase(owner, "stage", name) , AllocatedBytes(TBase::GetValue("Allocated/Bytes")) , AllocatedChunks(TBase::GetValue("Allocated/Count")) + , Allocations(TBase::GetDeriviative("Allocated/Count")) + , Free(TBase::GetDeriviative("Free/Count")) , WaitingBytes(TBase::GetValue("Waiting/Bytes")) , WaitingChunks(TBase::GetValue("Waiting/Count")) - , AllocationFailCount(TBase::GetValue("AllocationFails/Count")) { + , AllocationFailCount(TBase::GetValue("AllocationFails/Count")) + , ValueHardLimit(TBase::GetValue("Limit/Hard/Bytes")) + , ValueSoftLimit(TBase::GetValue("Limit/Soft/Bytes")) { } void OnCannotAllocate() { @@ -27,6 +36,7 @@ class TStageCounters: public NColumnShard::TCommonCountersOwner { } void Add(const ui64 volume, const bool allocated) { + Allocations->Inc(); if (allocated) { AllocatedBytes->Add(volume); AllocatedChunks->Add(1); @@ -37,6 +47,7 @@ class TStageCounters: public NColumnShard::TCommonCountersOwner { } void Sub(const ui64 volume, const bool allocated) { + Free->Inc(); if (allocated) { AllocatedBytes->Sub(volume); AllocatedChunks->Sub(1); diff --git a/ydb/core/tx/limiter/grouped_memory/service/process.h b/ydb/core/tx/limiter/grouped_memory/service/process.h index a7631dd390ef..524c40566576 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/process.h +++ b/ydb/core/tx/limiter/grouped_memory/service/process.h @@ -76,9 +76,9 @@ class TProcessMemoryScope: public NColumnShard::TMonitoringObjectsCounter internalGroupIdOptional = GroupIds.GetInternalIdOptional(externalGroupId); if (!internalGroupIdOptional) { - AFL_VERIFY(!task->OnAllocated(std::make_shared(ExternalProcessId, ExternalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( - "ext_group", externalGroupId)( - "min_group", GroupIds.GetMinInternalIdOptional())("stage", stage->GetName()); + AFL_VERIFY(!task->OnAllocated(std::make_shared(ExternalProcessId, ExternalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))("ext_group", externalGroupId)( + "min_int_group", GroupIds.GetMinInternalIdOptional())( + "min_ext_group", GroupIds.GetMinExternalIdOptional())("stage", stage->GetName()); AFL_VERIFY(!AllocationInfo.contains(task->GetIdentifier())); } else { const ui64 internalGroupId = *internalGroupIdOptional; diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index b662494d7b0b..2a18744bf612 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -36,7 +36,7 @@ class TServiceOperatorImpl { } else { AFL_VERIFY(Singleton()->DefaultStageFeatures); return std::make_shared( - name, limit, Max(), Singleton()->DefaultStageFeatures, Singleton()->Counters->BuildStageCounters(name)); + name, limit, std::nullopt, Singleton()->DefaultStageFeatures, Singleton()->Counters->BuildStageCounters(name)); } } @@ -100,4 +100,11 @@ class TScanMemoryLimiterPolicy { using TScanMemoryLimiterOperator = TServiceOperatorImpl; +class TCompMemoryLimiterPolicy { +public: + static const inline TString Name = "Comp"; +}; + +using TCompMemoryLimiterOperator = TServiceOperatorImpl; + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/usage/stage_features.cpp b/ydb/core/tx/limiter/grouped_memory/usage/stage_features.cpp index 2ad32c82030c..105b2b6eabff 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/stage_features.cpp +++ b/ydb/core/tx/limiter/grouped_memory/usage/stage_features.cpp @@ -15,13 +15,19 @@ TString TStageFeatures::DebugString() const { return result; } -TStageFeatures::TStageFeatures(const TString& name, const ui64 limit, const ui64 hardLimit, const std::shared_ptr& owner, - const std::shared_ptr& counters) +TStageFeatures::TStageFeatures(const TString& name, const ui64 limit, const std::optional& hardLimit, + const std::shared_ptr& owner, const std::shared_ptr& counters) : Name(name) , Limit(limit) , HardLimit(hardLimit) , Owner(owner) , Counters(counters) { + if (Counters) { + Counters->ValueSoftLimit->Set(Limit); + if (HardLimit) { + Counters->ValueHardLimit->Set(*HardLimit); + } + } } TConclusionStatus TStageFeatures::Allocate(const ui64 volume) { @@ -33,16 +39,16 @@ TConclusionStatus TStageFeatures::Allocate(const ui64 volume) { if (current->Counters) { current->Counters->Sub(volume, false); } - if (current->HardLimit < current->Usage.Val() + volume) { + if (current->HardLimit && *current->HardLimit < current->Usage.Val() + volume) { if (!result) { - result = TConclusionStatus::Fail(TStringBuilder() << current->Name << "::(limit:" << current->HardLimit + result = TConclusionStatus::Fail(TStringBuilder() << current->Name << "::(limit:" << *current->HardLimit << ";val:" << current->Usage.Val() << ";delta=" << volume << ");"); } if (current->Counters) { current->Counters->OnCannotAllocate(); } AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("name", current->Name)("event", "cannot_allocate")( - "limit", current->HardLimit)("usage", current->Usage.Val())("delta", volume); + "limit", *current->HardLimit)("usage", current->Usage.Val())("delta", volume); } current = current->Owner.get(); } diff --git a/ydb/core/tx/limiter/grouped_memory/usage/stage_features.h b/ydb/core/tx/limiter/grouped_memory/usage/stage_features.h index 4cb9146152f4..141923a40606 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/stage_features.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/stage_features.h @@ -10,7 +10,7 @@ class TStageFeatures { private: YDB_READONLY_DEF(TString, Name); YDB_READONLY(ui64, Limit, 0); - YDB_READONLY(ui64, HardLimit, 0); + YDB_READONLY_DEF(std::optional, HardLimit); YDB_ACCESSOR_DEF(TPositiveControlInteger, Usage); YDB_ACCESSOR_DEF(TPositiveControlInteger, Waiting); std::shared_ptr Owner; @@ -23,7 +23,7 @@ class TStageFeatures { return Usage.Val() + Waiting.Val(); } - TStageFeatures(const TString& name, const ui64 limit, const ui64 hardLimit, const std::shared_ptr& owner, + TStageFeatures(const TString& name, const ui64 limit, const std::optional& hardLimit, const std::shared_ptr& owner, const std::shared_ptr& counters); [[nodiscard]] TConclusionStatus Allocate(const ui64 volume); diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 9d9d50187c5b..6e3368b7045c 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -164,7 +164,8 @@ def __init__( separate_node_configs=False, default_clusteradmin=None, enable_resource_pools=None, - grouped_memory_limiter_config=None, + scan_grouped_memory_limiter_config=None, + comp_grouped_memory_limiter_config=None, query_service_config=None, domain_login_only=None, use_self_management=False, @@ -392,6 +393,14 @@ def __init__( if column_shard_config: self.yaml_config["column_shard_config"] = column_shard_config + if query_service_config: + self.yaml_config["query_service_config"] = query_service_config + + if scan_grouped_memory_limiter_config: + self.yaml_config["scan_grouped_memory_limiter_config"] = scan_grouped_memory_limiter_config + if comp_grouped_memory_limiter_config: + self.yaml_config["comp_grouped_memory_limiter_config"] = comp_grouped_memory_limiter_config + self.__build() if self.grpc_ssl_enable: From 8dbab39e19d68b3f1464bd271c405645adac7a56 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Wed, 18 Jun 2025 12:31:32 +0300 Subject: [PATCH 23/62] additional signals (#19745) Co-authored-by: ivanmorozov333 --- ydb/core/tx/columnshard/data_reader/fetcher.h | 46 ++++++- .../engines/changes/abstract/abstract.cpp | 25 ++-- .../engines/changes/abstract/abstract.h | 6 +- .../engines/changes/counters/changes.cpp | 3 +- .../engines/changes/counters/changes.h | 68 ++-------- ydb/library/actors/core/actorsystem.cpp | 3 - ydb/library/actors/core/actorsystem.h | 8 +- ydb/library/signals/states.cpp | 5 + ydb/library/signals/states.h | 116 ++++++++++++++++++ ydb/library/signals/ya.make | 1 + 10 files changed, 197 insertions(+), 84 deletions(-) create mode 100644 ydb/library/signals/states.cpp create mode 100644 ydb/library/signals/states.h diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index b0a191fc80ab..b94cfd5f71ad 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -11,19 +11,56 @@ #include #include +#include namespace NKikimr::NOlap::NDataFetcher { +class TClassCounters { +private: + std::shared_ptr> StateSignals; + +public: + TClassCounters(NColumnShard::TCommonCountersOwner& owner) { + StateSignals = std::make_shared>(owner, "fetching_stage"); + } + + NCounters::TStateSignalsOperator::TGuard GetGuard(const std::optional start) { + return StateSignals->BuildGuard(start); + } +}; + +class TCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + TMutex Mutex; + THashMap> ClassCounters; + +public: + TCounters() + : TBase("data_fetcher") { + } + + std::shared_ptr GetClassCounters(const TString& className) { + TGuard g(Mutex); + auto it = ClassCounters.find(className); + if (it == ClassCounters.end()) { + it = ClassCounters.emplace(className, std::make_shared(*this)).first; + } + return it->second; + } +}; + class TPortionsDataFetcher: TNonCopyable { private: const TRequestInput Input; const std::shared_ptr Callback; + std::shared_ptr ClassCounters; + NCounters::TStateSignalsOperator::TGuard Guard; TScriptExecution Script; TCurrentContext CurrentContext; std::shared_ptr Environment; const NConveyorComposite::ESpecialTaskCategory ConveyorCategory; bool IsFinishedFlag = false; - EFetchingStage Stage = EFetchingStage::Created; public: void AskMemoryAllocation(const std::shared_ptr& task) { @@ -35,6 +72,8 @@ class TPortionsDataFetcher: TNonCopyable { const std::shared_ptr& script, const NConveyorComposite::ESpecialTaskCategory conveyorCategory) : Input(std::move(input)) , Callback(std::move(callback)) + , ClassCounters(Singleton()->GetClassCounters(Callback->GetClassName())) + , Guard(ClassCounters->GetGuard(EFetchingStage::Created)) , Script(script) , Environment(environment) , ConveyorCategory(conveyorCategory) { @@ -43,7 +82,8 @@ class TPortionsDataFetcher: TNonCopyable { } ~TPortionsDataFetcher() { - AFL_VERIFY(IsFinishedFlag || Stage == EFetchingStage::Created || Callback->IsAborted())("stage", Stage)("class_name", Callback->GetClassName()); + AFL_VERIFY(NActors::TActorSystem::IsStopped() || IsFinishedFlag || Guard.GetStage() == EFetchingStage::Created + || Callback->IsAborted())("stage", Guard.GetStage())("class_name", Callback->GetClassName()); } static void StartAccessorPortionsFetching(TRequestInput&& input, std::shared_ptr&& callback, @@ -78,7 +118,7 @@ class TPortionsDataFetcher: TNonCopyable { void SetStage(const EFetchingStage stage) { Callback->OnStageStarting(stage); - Stage = stage; + Guard.SetState(stage); } void OnError(const TString& errMessage) { diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp index b8b89497ed86..f12fa3e99785 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp @@ -10,7 +10,7 @@ namespace NKikimr::NOlap { void TColumnEngineChanges::SetStage(const NChanges::EStage stage) { - Counters->SetStage(stage); + StateGuard.SetState(stage); } TString TColumnEngineChanges::DebugString() const { @@ -23,8 +23,8 @@ TString TColumnEngineChanges::DebugString() const { TConclusionStatus TColumnEngineChanges::ConstructBlobs(TConstructionContext& context) noexcept { const NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("task_id", GetTaskIdentifier())("task_class", TypeString()); - AFL_VERIFY(Counters->GetStage() == NChanges::EStage::ReadyForConstruct || Counters->GetStage() == NChanges::EStage::Started)( - "actual_stage", Counters->GetStage()); + AFL_VERIFY(StateGuard.GetStage() == NChanges::EStage::ReadyForConstruct || StateGuard.GetStage() == NChanges::EStage::Started)( + "actual_stage", StateGuard.GetStage()); context.Counters.CompactionInputSize(Blobs.GetTotalBlobsSize()); const TMonotonic start = TMonotonic::Now(); @@ -39,16 +39,16 @@ TConclusionStatus TColumnEngineChanges::ConstructBlobs(TConstructionContext& con } void TColumnEngineChanges::WriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { - AFL_VERIFY(Counters->GetStage() != NChanges::EStage::Aborted); - AFL_VERIFY(Counters->GetStage() <= NChanges::EStage::Written); - AFL_VERIFY(Counters->GetStage() >= NChanges::EStage::Compiled); + AFL_VERIFY(StateGuard.GetStage() != NChanges::EStage::Aborted); + AFL_VERIFY(StateGuard.GetStage() <= NChanges::EStage::Written); + AFL_VERIFY(StateGuard.GetStage() >= NChanges::EStage::Compiled); DoWriteIndexOnExecute(self, context); SetStage(NChanges::EStage::Written); } void TColumnEngineChanges::WriteIndexOnComplete(NColumnShard::TColumnShard* self, TWriteIndexCompleteContext& context) { - Y_ABORT_UNLESS(Counters->GetStage() == NChanges::EStage::Written || !self); + Y_ABORT_UNLESS(StateGuard.GetStage() == NChanges::EStage::Written || !self); SetStage(NChanges::EStage::Finished); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "WriteIndexComplete")("type", TypeString())("success", context.FinishedSuccessfully); DoWriteIndexOnComplete(self, context); @@ -59,7 +59,7 @@ void TColumnEngineChanges::WriteIndexOnComplete(NColumnShard::TColumnShard* self } void TColumnEngineChanges::Compile(TFinalizationContext& context) noexcept { - AFL_VERIFY(Counters->GetStage() != NChanges::EStage::Aborted); + AFL_VERIFY(StateGuard.GetStage() != NChanges::EStage::Aborted); DoCompile(context); DoOnAfterCompile(); @@ -73,7 +73,8 @@ TColumnEngineChanges::~TColumnEngineChanges() { void TColumnEngineChanges::Abort(NColumnShard::TColumnShard& self, TChangesFinishContext& context) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "Abort")("reason", context.ErrorMessage); - AFL_VERIFY(Counters->GetStage() != NChanges::EStage::Finished && Counters->GetStage() != NChanges::EStage::Created && Counters->GetStage() != NChanges::EStage::Aborted)("stage", Counters->GetStage())( + AFL_VERIFY(StateGuard.GetStage() != NChanges::EStage::Finished && StateGuard.GetStage() != NChanges::EStage::Created && StateGuard.GetStage() != NChanges::EStage::Aborted)( + "stage", StateGuard.GetStage())( "reason", context.ErrorMessage)("prev_reason", AbortedReason); SetStage(NChanges::EStage::Aborted); AbortedReason = context.ErrorMessage; @@ -83,7 +84,7 @@ void TColumnEngineChanges::Abort(NColumnShard::TColumnShard& self, TChangesFinis void TColumnEngineChanges::Start(NColumnShard::TColumnShard& self) { AFL_VERIFY(!LockGuard); LockGuard = self.DataLocksManager->RegisterLock(BuildDataLock()); - Y_ABORT_UNLESS(Counters->GetStage() == NChanges::EStage::Created); + Y_ABORT_UNLESS(StateGuard.GetStage() == NChanges::EStage::Created); NYDBTest::TControllers::GetColumnShardController()->OnWriteIndexStart(self.TabletID(), *this); DoStart(self); SetStage(NChanges::EStage::Started); @@ -93,7 +94,7 @@ void TColumnEngineChanges::Start(NColumnShard::TColumnShard& self) { } void TColumnEngineChanges::StartEmergency() { - Y_ABORT_UNLESS(Counters->GetStage() == NChanges::EStage::Created); + Y_ABORT_UNLESS(StateGuard.GetStage() == NChanges::EStage::Created); SetStage(NChanges::EStage::Started); // if (!NeedConstruction()) { // SetStage(NChanges::EStage::Constructed); @@ -102,7 +103,7 @@ void TColumnEngineChanges::StartEmergency() { void TColumnEngineChanges::AbortEmergency(const TString& reason) { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "AbortEmergency")("reason", reason)("prev_reason", AbortedReason); - if (Counters->GetStage() == NChanges::EStage::Aborted) { + if (StateGuard.GetStage() == NChanges::EStage::Aborted) { AbortedReason += "; AnotherReason: " + reason; } else { SetStage(NChanges::EStage::Aborted); diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index b478eda7c637..6bf35c6dd29e 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -255,7 +255,7 @@ class TColumnEngineChanges: public TMoveOnly { TString AbortedReason; const TString TaskIdentifier = TGUID::CreateTimebased().AsGuidString(); std::shared_ptr ActivityFlag; - std::shared_ptr Counters; + NCounters::TStateSignalsOperator::TGuard StateGuard; protected: std::optional FetchedDataAccessors; @@ -354,7 +354,7 @@ class TColumnEngineChanges: public TMoveOnly { } TColumnEngineChanges(const std::shared_ptr& storagesManager, const NBlobOperations::EConsumer consumerId) - : Counters(NChanges::TChangesCounters::GetStageCounters(consumerId)) + : StateGuard(NChanges::TChangesCounters::GetStageCounters(consumerId)) , BlobsAction(storagesManager, consumerId) { } @@ -362,7 +362,7 @@ class TColumnEngineChanges: public TMoveOnly { virtual ~TColumnEngineChanges(); bool IsAborted() const { - return Counters->GetCurrentStage() == NChanges::EStage::Aborted; + return StateGuard.GetStage() == NChanges::EStage::Aborted; } void StartEmergency(); diff --git a/ydb/core/tx/columnshard/engines/changes/counters/changes.cpp b/ydb/core/tx/columnshard/engines/changes/counters/changes.cpp index 5918d35274e4..e6057670b69f 100644 --- a/ydb/core/tx/columnshard/engines/changes/counters/changes.cpp +++ b/ydb/core/tx/columnshard/engines/changes/counters/changes.cpp @@ -1,3 +1,4 @@ #include "changes.h" -namespace NKikimr::NOlap::NChanges {} +namespace NKikimr::NOlap::NChanges { +} diff --git a/ydb/core/tx/columnshard/engines/changes/counters/changes.h b/ydb/core/tx/columnshard/engines/changes/counters/changes.h index 6f1ab42ec698..0c1a0dcfa31d 100644 --- a/ydb/core/tx/columnshard/engines/changes/counters/changes.h +++ b/ydb/core/tx/columnshard/engines/changes/counters/changes.h @@ -1,9 +1,10 @@ #pragma once #include -#include #include +#include +#include #include #include @@ -60,77 +61,26 @@ class TChangesCounters: public NColumnShard::TCommonCountersOwner { } }; - class TStageCounters: NColumnShard::TCommonCountersOwner { - private: - using TBase = NColumnShard::TCommonCountersOwner; - std::array, (ui64)EStage::COUNT> Stages; - - public: - TStageCounters(const NColumnShard::TCommonCountersOwner& owner, const NBlobOperations::EConsumer consumerId) - : TBase(owner, "consumer", ToString(consumerId)) { - for (size_t i = 0; i < (ui64)EStage::COUNT; ++i) { - Stages[i] = std::make_shared(TBase::CreateSubGroup("stage", ToString(static_cast(i)))); - } - } - - void OnStageChanged(const std::optional stageFrom, const std::optional stageTo) const { - if (stageFrom) { - AFL_VERIFY(static_cast(*stageFrom) < Stages.size())("index", *stageFrom)("size", Stages.size()); - Stages[static_cast(*stageFrom)]->Dec(); - } - if (stageTo) { - AFL_VERIFY(static_cast(*stageTo) < Stages.size())("index", *stageTo)("size", Stages.size()); - Stages[static_cast(*stageTo)]->Inc(); - } - } - }; - private: - std::array, static_cast(NBlobOperations::EConsumer::COUNT)> StagesByConsumer; + std::array>, static_cast(NBlobOperations::EConsumer::COUNT)> StagesByConsumer; - std::shared_ptr GetStageCountersImpl(const NBlobOperations::EConsumer consumerId) { + std::shared_ptr> GetStageCountersImpl(const NBlobOperations::EConsumer consumerId) { AFL_VERIFY((ui64)consumerId < StagesByConsumer.size())("index", consumerId)("size", StagesByConsumer.size()); return StagesByConsumer[(ui64)consumerId]; } public: - class TStageCountersGuard: TNonCopyable { - private: - const std::shared_ptr Counters; - YDB_READONLY(EStage, CurrentStage, EStage::Created); - public: - EStage GetStage() const { - return CurrentStage; - } - - TStageCountersGuard(const std::shared_ptr& counters, const EStage startStage) - : Counters(counters) - , CurrentStage(startStage) { - Counters->OnStageChanged(std::nullopt, startStage); - } - - ~TStageCountersGuard() { - Counters->OnStageChanged(CurrentStage, std::nullopt); - } - - void SetStage(const EStage stageTo) { - AFL_VERIFY(stageTo >= CurrentStage)("current", CurrentStage)("to", stageTo); - if (CurrentStage != stageTo) { - Counters->OnStageChanged(CurrentStage, stageTo); - CurrentStage = stageTo; - } - } - }; - TChangesCounters() : TBase("ColumnEngineChanges") { for (ui64 i = 0; i < (ui64)NBlobOperations::EConsumer::COUNT; ++i) { - StagesByConsumer[i] = std::make_shared(*this, static_cast(i)); + auto base = this->CreateSubGroup("consumer", ::ToString(static_cast(i))); + StagesByConsumer[i] = std::make_shared>( + base, "indexation_stage"); } } - static std::shared_ptr GetStageCounters(const NBlobOperations::EConsumer consumerId) { - return std::make_shared(Singleton()->GetStageCountersImpl(consumerId), EStage::Created); + static NCounters::TStateSignalsOperator::TGuard GetStageCounters(const NBlobOperations::EConsumer consumerId) { + return Singleton()->GetStageCountersImpl(consumerId)->BuildGuard(EStage::Created); } }; diff --git a/ydb/library/actors/core/actorsystem.cpp b/ydb/library/actors/core/actorsystem.cpp index 1f891f787fc2..f964154dda53 100644 --- a/ydb/library/actors/core/actorsystem.cpp +++ b/ydb/library/actors/core/actorsystem.cpp @@ -84,9 +84,6 @@ namespace NActors { , DefSelfID(NodeId, "actorsystem") , AppData0(appData) , LoggerSettings0(loggerSettings) - , StartExecuted(false) - , StopExecuted(false) - , CleanupExecuted(false) { ServiceMap.Reset(new TServiceMap()); } diff --git a/ydb/library/actors/core/actorsystem.h b/ydb/library/actors/core/actorsystem.h index dd2ee8a7839e..05e9027ac72b 100644 --- a/ydb/library/actors/core/actorsystem.h +++ b/ydb/library/actors/core/actorsystem.h @@ -162,9 +162,9 @@ namespace NActors { TMutex ProxyCreationLock; mutable std::vector DynamicProxies; - bool StartExecuted; - bool StopExecuted; - bool CleanupExecuted; + bool StartExecuted = false; + bool StopExecuted = false; + bool CleanupExecuted = false; std::deque> DeferredPreStop; public: @@ -176,6 +176,8 @@ namespace NActors { void Stop(); void Cleanup(); + static bool IsStopped(); + template TActorId Register(IActor* actor, TMailboxType::EType mailboxType = TMailboxType::HTSwap, ui32 executorPool = 0, ui64 revolvingCounter = 0, const TActorId& parentId = TActorId()); diff --git a/ydb/library/signals/states.cpp b/ydb/library/signals/states.cpp new file mode 100644 index 000000000000..62971f492161 --- /dev/null +++ b/ydb/library/signals/states.cpp @@ -0,0 +1,5 @@ +#include "states.h" + +namespace NKikimr::NOlap::NCounters { + +} \ No newline at end of file diff --git a/ydb/library/signals/states.h b/ydb/library/signals/states.h new file mode 100644 index 000000000000..ef7e17648dfc --- /dev/null +++ b/ydb/library/signals/states.h @@ -0,0 +1,116 @@ +#pragma once +#include "owner.h" + +#include + +#include +#include + +#include +#include + +namespace NKikimr::NOlap::NCounters { + +class TStateCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + +public: + const NMonitoring::TDynamicCounters::TCounterPtr Volume; + const NMonitoring::TDynamicCounters::TCounterPtr Add; + const NMonitoring::TDynamicCounters::TCounterPtr Remove; + const NMonitoring::TDynamicCounters::TCounterPtr Duration; + + TStateCounters(TCommonCountersOwner& base, const TString& stateId) + : TBase(base, "state_id", stateId) + , Volume(TBase::GetValue("Count")) + , Add(TBase::GetDeriviative("Add")) + , Remove(TBase::GetDeriviative("Remove")) + , Duration(TBase::GetDeriviative("Duration/Us")) { + } +}; + +template +class TStateSignalsOwner: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + std::vector States; + +public: + void ExchangeState(const std::optional from, const std::optional to, const std::optional d, const ui32 size = 1) { + if (from) { + AFL_VERIFY((ui32)*from < States.size())("from", from)("size", States.size()); + States[(ui32)*from].Volume->Sub(size); + States[(ui32)*from].Remove->Add(size); + if (d) { + States[(ui32)*from].Duration->Add(d->MicroSeconds()); + } + } + if (to) { + AFL_VERIFY((ui32)*to < States.size())("to", to)("size", States.size()); + States[(ui32)*to].Volume->Add(size); + States[(ui32)*to].Add->Add(size); + } + } + + TStateSignalsOwner(TCommonCountersOwner& base, const TString& stateName) + : TBase(base, "states", stateName) { + for (auto&& i : GetEnumAllValues()) { + while (States.size() < (ui32)i) { + States.emplace_back(*this, "UNDEFINED"); + } + States.emplace_back(*this, ::ToString(i)); + } + } +}; + +template +class TStateSignalsOperator { +private: + std::shared_ptr> Signals; + +public: + class TGuard: public TMoveOnly { + private: + std::optional CurrentStateInstant; + std::optional CurrentState; + std::shared_ptr> Signals; + + public: + EState GetStage() const { + AFL_VERIFY(CurrentState); + return *CurrentState; + } + + void SetState(const std::optional state) { + std::optional d; + const TMonotonic current = TMonotonic::Now(); + if (CurrentStateInstant) { + d = current - *CurrentStateInstant; + } + Signals->ExchangeState(CurrentState, state, d); + CurrentState = state; + CurrentStateInstant = current; + } + + TGuard(const std::shared_ptr>& owner, const std::optional start) + : Signals(owner) + { + SetState(start); + } + + ~TGuard() { + SetState(std::nullopt); + } + }; + + TStateSignalsOperator(NColumnShard::TCommonCountersOwner& base, const TString& stateName) { + Signals = std::make_shared>(base, stateName); + } + + TGuard BuildGuard(const std::optional startState) { + return TGuard(Signals, startState); + } +}; + +} // namespace NKikimr::NOlap::NCounters diff --git a/ydb/library/signals/ya.make b/ydb/library/signals/ya.make index b2e330fc8b9b..fcf4832399cb 100644 --- a/ydb/library/signals/ya.make +++ b/ydb/library/signals/ya.make @@ -7,6 +7,7 @@ SRCS( private.cpp object_counter.cpp histogram.cpp + states.cpp ) PEERDIR( From 0e1008edde267c4d1c47c879e0fc4d7e6cae6190 Mon Sep 17 00:00:00 2001 From: r314-git Date: Wed, 18 Jun 2025 17:06:12 +0300 Subject: [PATCH 24/62] Add operation increment in datashard (#19567) --- ydb/core/io_formats/cell_maker/cell_maker.cpp | 58 ++++ ydb/core/io_formats/cell_maker/cell_maker.h | 2 + ydb/core/protos/data_events.proto | 1 + ydb/core/scheme/scheme_tablecell.cpp | 9 + ydb/core/scheme/scheme_tablecell.h | 2 - .../scheme/versions/abstract_scheme.cpp | 1 + .../operations/batch_builder/builder.cpp | 4 + .../tx/data_events/common/modification_type.h | 13 +- ydb/core/tx/data_events/write_data.h | 1 + .../tx/datashard/datashard__engine_host.cpp | 4 + ydb/core/tx/datashard/datashard_user_db.cpp | 71 ++++- ydb/core/tx/datashard/datashard_user_db.h | 11 + ydb/core/tx/datashard/datashard_ut_write.cpp | 264 ++++++++++++++++++ .../datashard/datashard_write_operation.cpp | 23 ++ ydb/core/tx/datashard/execute_write_unit.cpp | 8 +- .../ut_common/datashard_ut_common.cpp | 40 ++- .../datashard/ut_common/datashard_ut_common.h | 3 + ydb/core/tx/datashard/ya.make | 1 + 18 files changed, 508 insertions(+), 8 deletions(-) diff --git a/ydb/core/io_formats/cell_maker/cell_maker.cpp b/ydb/core/io_formats/cell_maker/cell_maker.cpp index 5a50434ae88d..e1b07e187ade 100644 --- a/ydb/core/io_formats/cell_maker/cell_maker.cpp +++ b/ydb/core/io_formats/cell_maker/cell_maker.cpp @@ -1,5 +1,6 @@ #include "cell_maker.h" +#include #include #include #include @@ -282,6 +283,63 @@ namespace { } // anonymous +void AddTwoCells(TCell& result, const TCell& cell1, const TCell& cell2, const NScheme::TTypeId& typeId) { + + Y_ENSURE(cell1.Size() == NScheme::GetFixedSize(typeId)); + Y_ENSURE(cell2.Size() == NScheme::GetFixedSize(typeId)); + + switch (typeId) { + case NScheme::NTypeIds::Int8: + result = TCell::Make(i8(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Uint8: + result = TCell::Make(ui8(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Int16: + result = TCell::Make(i16(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Uint16: + result = TCell::Make(ui16(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Int32: + result = TCell::Make(i32(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Uint32: + result = TCell::Make(ui32(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Int64: + result = TCell::Make(i64(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Uint64: + result = TCell::Make(ui64(cell1.AsValue() + cell2.AsValue())); + break; + case NScheme::NTypeIds::Float: + case NScheme::NTypeIds::Double: + case NScheme::NTypeIds::Date: + case NScheme::NTypeIds::Datetime: + case NScheme::NTypeIds::Timestamp: + case NScheme::NTypeIds::Interval: + case NScheme::NTypeIds::Date32: + case NScheme::NTypeIds::Datetime64: + case NScheme::NTypeIds::Timestamp64: + case NScheme::NTypeIds::Interval64: + case NScheme::NTypeIds::String: + case NScheme::NTypeIds::String4k: + case NScheme::NTypeIds::String2m: + case NScheme::NTypeIds::Utf8: + case NScheme::NTypeIds::Yson: + case NScheme::NTypeIds::Json: + case NScheme::NTypeIds::JsonDocument: + case NScheme::NTypeIds::DyNumber: + case NScheme::NTypeIds::Decimal: + case NScheme::NTypeIds::Pg: + case NScheme::NTypeIds::Uuid: + Y_ENSURE(false); + default: + Y_ENSURE(false); + } +} + bool MakeCell(TCell& cell, TStringBuf value, const NScheme::TTypeInfo& typeInfo, TMemoryPool& pool, TString& err) { if (value == "null") { return true; diff --git a/ydb/core/io_formats/cell_maker/cell_maker.h b/ydb/core/io_formats/cell_maker/cell_maker.h index 8c9f2d66b87b..60a3af7b3a1e 100644 --- a/ydb/core/io_formats/cell_maker/cell_maker.h +++ b/ydb/core/io_formats/cell_maker/cell_maker.h @@ -13,6 +13,8 @@ namespace NKikimr::NFormats { bool MakeCell(TCell& cell, TStringBuf value, const NScheme::TTypeInfo& typeInfo, TMemoryPool& pool, TString& err); bool MakeCell(TCell& cell, const NJson::TJsonValue& value, const NScheme::TTypeInfo& typeInfo, TMemoryPool& pool, TString& err); +void AddTwoCells(TCell& result, const TCell& cell1, const TCell& cell2, const NScheme::TTypeId& typeId); + bool CheckCellValue(const TCell& cell, const NScheme::TTypeInfo& typeInfo); } diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto index cc505d86a9f6..0010bd0c6d0f 100644 --- a/ydb/core/protos/data_events.proto +++ b/ydb/core/protos/data_events.proto @@ -93,6 +93,7 @@ message TEvWrite { OPERATION_UPDATE = 3; OPERATION_UPSERT = 4; OPERATION_REPLACE = 5; + OPERATION_INCREMENT = 6; } optional EOperationType Type = 1; diff --git a/ydb/core/scheme/scheme_tablecell.cpp b/ydb/core/scheme/scheme_tablecell.cpp index c376e61eb83f..21c8bb948c0d 100644 --- a/ydb/core/scheme/scheme_tablecell.cpp +++ b/ydb/core/scheme/scheme_tablecell.cpp @@ -675,9 +675,18 @@ void DbgPrintValue(TString &res, const TCell &r, NScheme::TTypeInfo typeInfo) { case NScheme::NTypeIds::Bool: res += r.AsValue() ? "true" : "false"; break; + case NScheme::NTypeIds::Int8: + res += ToString(r.AsValue()); + break; case NScheme::NTypeIds::Byte: res += ToString(r.AsValue()); break; + case NScheme::NTypeIds::Int16: + res += ToString(r.AsValue()); + break; + case NScheme::NTypeIds::Uint16: + res += ToString(r.AsValue()); + break; case NScheme::NTypeIds::Int32: res += ToString(r.AsValue()); break; diff --git a/ydb/core/scheme/scheme_tablecell.h b/ydb/core/scheme/scheme_tablecell.h index 9d56525dd2d6..a53df76f1a99 100644 --- a/ydb/core/scheme/scheme_tablecell.h +++ b/ydb/core/scheme/scheme_tablecell.h @@ -1,7 +1,6 @@ #pragma once #include "defs.h" -#include "scheme_type_id.h" #include "scheme_type_order.h" #include "scheme_types_defs.h" @@ -11,7 +10,6 @@ #include #include -#include #include namespace NKikimr { diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp index 049e0109fe81..b051d3ecff20 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp @@ -125,6 +125,7 @@ TConclusion> ISnapshotSchema:: } case NEvWrite::EModificationType::Delete: case NEvWrite::EModificationType::Update: + case NEvWrite::EModificationType::Increment: break; } } diff --git a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp index ddf8628b80c8..e9bd1fea1e0d 100644 --- a/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/batch_builder/builder.cpp @@ -83,6 +83,10 @@ void TBuildBatchesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { merger = std::make_shared(batch, Context.GetActualSchema(), ""); break; } + case NEvWrite::EModificationType::Increment: { + merger = std::make_shared(batch, Context.GetActualSchema(), ""); + break; + } case NEvWrite::EModificationType::Replace: case NEvWrite::EModificationType::Delete: { if (!Context.GetNoTxWrite()) { diff --git a/ydb/core/tx/data_events/common/modification_type.h b/ydb/core/tx/data_events/common/modification_type.h index f93eeda183e3..ec9dc131bb69 100644 --- a/ydb/core/tx/data_events/common/modification_type.h +++ b/ydb/core/tx/data_events/common/modification_type.h @@ -1,6 +1,8 @@ #pragma once +#include "util/generic/yexception.h" #include #include +#include namespace NKikimr::NEvWrite { enum class EModificationType { @@ -8,7 +10,8 @@ enum class EModificationType { Insert, Update, Replace, - Delete + Delete, + Increment }; } @@ -30,6 +33,7 @@ class TEnumOperator { case NEvWrite::EModificationType::Upsert: case NEvWrite::EModificationType::Delete: case NEvWrite::EModificationType::Update: + case NEvWrite::EModificationType::Increment: return false; case NEvWrite::EModificationType::Insert: case NEvWrite::EModificationType::Replace: @@ -42,6 +46,7 @@ class TEnumOperator { case NEvWrite::EModificationType::Upsert: case NEvWrite::EModificationType::Delete: case NEvWrite::EModificationType::Update: + case NEvWrite::EModificationType::Increment: return false; case NEvWrite::EModificationType::Insert: case NEvWrite::EModificationType::Replace: @@ -61,6 +66,8 @@ class TEnumOperator { return NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE; case NEvWrite::EModificationType::Update: return NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPDATE; + case NEvWrite::EModificationType::Increment: + return NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT; } } @@ -78,6 +85,8 @@ class TEnumOperator { return NEvWrite::EModificationType::Delete; case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE: return NEvWrite::EModificationType::Replace; + case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT: + return NEvWrite::EModificationType::Increment; } } @@ -93,6 +102,8 @@ class TEnumOperator { return NKikimrTxColumnShard::TEvWrite::OPERATION_REPLACE; case NEvWrite::EModificationType::Update: return NKikimrTxColumnShard::TEvWrite::OPERATION_UPDATE; + case NEvWrite::EModificationType::Increment: + Y_ENSURE(false); } } diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index 409d48fd247d..a3c58ca85242 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -84,6 +84,7 @@ class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TN return true; case EModificationType::Update: case EModificationType::Replace: + case EModificationType::Increment: return false; } } diff --git a/ydb/core/tx/datashard/datashard__engine_host.cpp b/ydb/core/tx/datashard/datashard__engine_host.cpp index bf5e21831e4b..192ed73592b7 100644 --- a/ydb/core/tx/datashard/datashard__engine_host.cpp +++ b/ydb/core/tx/datashard/datashard__engine_host.cpp @@ -387,6 +387,10 @@ class TDataShardEngineHost final UserDb.UpdateRow(tableId, key, ops); } + void IncrementRow(const TTableId& tableId, const TArrayRef key, const TArrayRef ops) override { + UserDb.IncrementRow(tableId, key, ops); + } + void EraseRow(const TTableId& tableId, const TArrayRef& row) override { if (TSysTables::IsSystemTable(tableId)) { DataShardSysTable(tableId).EraseRow(row); diff --git a/ydb/core/tx/datashard/datashard_user_db.cpp b/ydb/core/tx/datashard/datashard_user_db.cpp index dea44ca9066d..bfe898b4ca05 100644 --- a/ydb/core/tx/datashard/datashard_user_db.cpp +++ b/ydb/core/tx/datashard/datashard_user_db.cpp @@ -1,6 +1,8 @@ #include "datashard_user_db.h" #include "datashard_impl.h" +#include +#include namespace NKikimr::NDataShard { @@ -177,6 +179,50 @@ void TDataShardUserDb::UpdateRow( IncreaseUpdateCounters(key, ops); } +void TDataShardUserDb::IncrementRow( + const TTableId& tableId, + const TArrayRef key, + const TArrayRef ops) +{ + auto localTableId = Self.GetLocalTableId(tableId); + Y_ENSURE(localTableId != 0, "Unexpected incrementRow for an unknown table"); + + TStackVec columns(ops.size()); + for (size_t i = 0; i < ops.size(); i++) { + columns[i] = ops[i].Tag; + } + + auto currentRow = GetRowState(tableId, key, columns); + + if (currentRow.Size() == 0) { + return; + } + + TStackVec newOps(ops.size()); + + Y_ENSURE(currentRow.Size() == ops.size()); + const NTable::TScheme& scheme = Db.GetScheme(); + const NTable::TScheme::TTableInfo* tableInfo = scheme.GetTableInfo(localTableId); + + TStackVec incrementResults(ops.size()); + + for (size_t i = 0; i < ops.size(); i++) { + auto vtype = scheme.GetColumnInfo(tableInfo, ops[i].Tag)->PType.GetTypeId(); + + auto current = currentRow.Get(i); + auto delta = ops[i].AsCell(); + + NFormats::AddTwoCells(incrementResults[i], current, delta, vtype); + + TRawTypeValue rawTypeValue(incrementResults[i].Data(), incrementResults[i].Size(), vtype); + newOps[i] = NIceDb::TUpdateOp(ops[i].Tag, ops[i].Op, rawTypeValue); + } + + UpsertRowInt(NTable::ERowOp::Upsert, tableId, localTableId, key, newOps); + + IncreaseUpdateCounters(key, ops); +} + void TDataShardUserDb::EraseRow( const TTableId& tableId, const TArrayRef key) @@ -257,9 +303,9 @@ void TDataShardUserDb::UpsertRowInt( Self.GetKeyAccessSampler()->AddSample(tableId, keyCells); } -bool TDataShardUserDb::RowExists ( +bool TDataShardUserDb::RowExists( const TTableId& tableId, - const TArrayRef key) + const TArrayRef key) { NTable::TRowState rowState; const auto ready = SelectRow(tableId, key, {}, rowState); @@ -276,6 +322,27 @@ bool TDataShardUserDb::RowExists ( } } +NTable::TRowState TDataShardUserDb::GetRowState( + const TTableId& tableId, + const TArrayRef key, + const TStackVec& columns) +{ + NTable::TRowState rowState; + + const auto ready = SelectRow(tableId, key, columns, rowState); + switch (ready) { + case NTable::EReady::Page: { + throw TNotReadyTabletException(); + } + case NTable::EReady::Data: { + return std::move(rowState); + } + case NTable::EReady::Gone: { + return NTable::TRowState(); + } + } +} + TSmallVec TDataShardUserDb::ConvertTableKeys(const TArrayRef key) { TSmallVec keyCells; diff --git a/ydb/core/tx/datashard/datashard_user_db.h b/ydb/core/tx/datashard/datashard_user_db.h index 013e34fe8461..b88518ac0f74 100644 --- a/ydb/core/tx/datashard/datashard_user_db.h +++ b/ydb/core/tx/datashard/datashard_user_db.h @@ -57,6 +57,11 @@ class IDataShardUserDb { const TArrayRef key, const TArrayRef ops) = 0; + virtual void IncrementRow( + const TTableId& tableId, + const TArrayRef key, + const TArrayRef ops) = 0; + virtual void EraseRow( const TTableId& tableId, const TArrayRef key) = 0; @@ -129,6 +134,11 @@ class TDataShardUserDb final const TTableId& tableId, const TArrayRef key, const TArrayRef ops) override; + + void IncrementRow( + const TTableId& tableId, + const TArrayRef key, + const TArrayRef ops) override; void EraseRow( const TTableId& tableId, @@ -182,6 +192,7 @@ class TDataShardUserDb final void UpsertRowInt(NTable::ERowOp rowOp, const TTableId& tableId, ui64 localTableId, const TArrayRef key, const TArrayRef ops); bool RowExists(const TTableId& tableId, const TArrayRef key); + NTable::TRowState GetRowState(const TTableId& tableId, const TArrayRef key, const TStackVec& columns); void IncreaseUpdateCounters(const TArrayRef key, const TArrayRef ops); private: diff --git a/ydb/core/tx/datashard/datashard_ut_write.cpp b/ydb/core/tx/datashard/datashard_ut_write.cpp index 8d92af4b989b..5ce5880880ef 100644 --- a/ydb/core/tx/datashard/datashard_ut_write.cpp +++ b/ydb/core/tx/datashard/datashard_ut_write.cpp @@ -16,6 +16,7 @@ using namespace Tests; Y_UNIT_TEST_SUITE(DataShardWrite) { + constexpr i16 operator""_i16(unsigned long long val) { return static_cast(val); } constexpr i32 operator""_i32(unsigned long long val) { return static_cast(val); } constexpr ui32 operator""_ui32(unsigned long long val) { return static_cast(val); } @@ -68,6 +69,269 @@ Y_UNIT_TEST_SUITE(DataShardWrite) { } } + Y_UNIT_TEST(IncrementImmediate) { + + auto [runtime, server, sender] = TestCreateServer(); + + // Define a table with different column types + auto opts = TShardedTableOptions() + .Columns({ + {"key", "Uint64", true, false}, // key (id=1) + {"uint8_val", "Uint8", false, false}, // id=2 + {"uint16_val", "Uint16", false, false}, // id=3 + {"uint32_val", "Uint32", false, false}, // id=4 + {"uint64_val", "Uint64", false, false}, // id=5 + {"int8_val", "Int8", false, false}, // id=6 + {"int16_val", "Int16", false, false}, // id=7 + {"int32_val", "Int32", false, false}, // id=8 + {"int64_val", "Int64", false, false}, // id=9 + {"utf8_val", "Utf8", false, false}, // id=10 (not numerical) + {"double_val", "Double", false, false} // id=11 (not supported by increment) + }); + + auto [shards, tableId] = CreateShardedTable(server, sender, "/Root", "table-1", opts); + const ui64 shard = shards[0]; + ui64 txId = 100; + + Cout << "========= Insert initial data =========\n"; + { + TVector columnIds = {1, 2, 3, 4, 5, 6, 7,8,9,10,11}; // all columns + TVector cells = { + TCell::Make(ui64(1)), // key = 1 + TCell::Make(ui8(10)), // uint8_val + TCell::Make(ui16(100)), // uint16_val + TCell::Make(ui32(1000)), // uint32_val + TCell::Make(ui64(10000)), // uint64_val + TCell::Make(i8(-10)), // int8_val + TCell::Make(i16(-50)), // int16_val + TCell::Make(i32(-500)), // int32_val + TCell::Make(i64(-5000)), // int64_val + TCell::Make("text"), // utf8_val + TCell::Make(3.14), // double_val + + TCell::Make(ui64(3)), // key = 3 + TCell::Make(ui8(15)), // uint8_val + TCell::Make(ui16(150)), // uint16_val + TCell::Make(ui32(1500)), // uint32_val + TCell::Make(ui64(15000)), // uint64_val + TCell::Make(i8(-15)), // int8_val + TCell::Make(i16(-55)), // int16_val + TCell::Make(i32(-550)), // int32_val + TCell::Make(i64(-5500)), // int64_val + TCell::Make("othertext"), // utf8_val + TCell::Make(3.15) // double_val + }; + + auto result = Upsert(runtime, sender, shard, tableId, txId, NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, cells); + + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + } + + Cout << "========= Verify initial data =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 10, uint16_val = 100, uint32_val = 1000, " + "uint64_val = 10000, int8_val = -10, int16_val = -50, int32_val = -500, int64_val = -5000, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Increment numeric columns =========\n"; + { + TVector columnIds = {1, 2, 3, 4, 5, 6, 7, 8, 9}; // key and numerical columns + TVector increments = { + TCell::Make(ui64(1)), // key + TCell::Make(ui8(5)), // +5 к uint8_val + TCell::Make(ui16(50)), // +50 к uint16_val + TCell::Make(ui32(500)), // +500 к uint32_val + TCell::Make(ui64(5000)), // +5000 к uint64_val + TCell::Make(i8(5)), // +5 к int8_val + TCell::Make(i16(10)), // +50 к int16_val + TCell::Make(i32(100)), // +100 к int32_val + TCell::Make(i64(1000)) // +1000 к int64_val + }; + + auto result = Increment(runtime, sender, shard, tableId, txId, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + } + + Cout << "========= Verify increment results =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -5, int16_val = -40, int32_val = -400, int64_val = -4000, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Try increment several rows (one don't exist) =========\n"; + { + TVector columnIds = {1,2}; //key column and uint8_val column + TVector increments = {TCell::Make(ui64(7)), TCell::Make(ui8(3)), // id 7 don't exist + TCell::Make(ui64(1)), TCell::Make(ui8(3))}; // id 1 exist + + auto result = Increment(runtime, sender, shard, tableId, txId, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + } + + Cout << "========= Verify data changed after increments =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 18, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -5, int16_val = -40, int32_val = -400, int64_val = -4000, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Try increment utf-8 column (should fail) =========\n"; + { + TVector columnIds = {1, 10}; // id, utf8_val + TVector increments = { + TCell::Make(ui64(1)), + TCell::Make("new_text"), + }; + + auto result = Increment(runtime, sender, shard, tableId, txId++, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + } + + Cout << "========= Try increment double column (should fail) =========\n"; + { + TVector columnIds = {1, 11}; // id, double_val + TVector increments = { + TCell::Make(ui64(1)), + TCell::Make(double(1.0)) + }; + + auto result = Increment(runtime, sender, shard, tableId, txId++, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + } + + Cout << "========= Verify data remains unchanged after failed increments =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 18, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -5, int16_val = -40, int32_val = -400, int64_val = -4000, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Try increment key columns =========\n"; + { + TVector columnIds = {1, 1}; // id, id + TVector increments = { + TCell::Make(ui64(1)), + TCell::Make(ui64(3)) + }; + + auto result = Increment(runtime, sender, shard, tableId, txId++, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST); + } + + Cout << "========= Verify data remains unchanged =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 18, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -5, int16_val = -40, int32_val = -400, int64_val = -4000, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Try increment with overflow =========\n"; + { + TVector columnIds = {1, 2, 3, 4, 5, 6, 7, 8, 9}; // id, id + TVector increments = { + TCell::Make(ui64(1)), // key + TCell::Make(ui8(~0)), // +2^8 - 1 + TCell::Make(ui16(~0)), // +2^16 - 1 + TCell::Make(ui32(~0)), // +2^32 - 1 + TCell::Make(ui64(~0ll)), // +2^64 - 1 + TCell::Make(i8(-((1 << 7) - 1))), // - (2^7-1) + TCell::Make(i16(-((1 << 15) - 1))), // - (2^15-1) + TCell::Make(i32(-((1 << 31) - 1))), // - (2^31-1) + TCell::Make(i64(-((1ll << 63) - 1))) // - (2^63-1) + }; + + auto result = Increment(runtime, sender, shard, tableId, txId++, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments, NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + } + + Cout << "========= Verify data =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 17, uint16_val = 149, uint32_val = 1499, " + "uint64_val = 14999, int8_val = 124, int16_val = 32729, int32_val = 2147483249, int64_val = 9223372036854771809, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + + Cout << "========= Try increment no delta columns =========\n"; + { + TVector columnIds = {1}; // id + TVector increments = {TCell::Make(ui64(1))}; + + auto result = Increment(runtime, sender, shard, tableId, txId++, + NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, columnIds, increments, NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + UNIT_ASSERT(result.GetStatus() == NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); + } + + Cout << "========= Verify data =========\n"; + { + auto tableState = ReadTable(server, shards, tableId); + + UNIT_ASSERT_STRINGS_EQUAL(tableState, + "key = 1, uint8_val = 17, uint16_val = 149, uint32_val = 1499, " + "uint64_val = 14999, int8_val = 124, int16_val = 32729, int32_val = 2147483249, int64_val = 9223372036854771809, " + "utf8_val = text\\0, double_val = 3.14\n" + + "key = 3, uint8_val = 15, uint16_val = 150, uint32_val = 1500, " + "uint64_val = 15000, int8_val = -15, int16_val = -55, int32_val = -550, int64_val = -5500, " + "utf8_val = othertext\\0, double_val = 3.15\n" + ); + } + } + Y_UNIT_TEST_QUAD(ExecSQLUpsertPrepared, EvWrite, Volatile) { NKikimrConfig::TAppConfig app; app.MutableTableServiceConfig()->SetEnableOltpSink(EvWrite); diff --git a/ydb/core/tx/datashard/datashard_write_operation.cpp b/ydb/core/tx/datashard/datashard_write_operation.cpp index 6d79110ad3ce..be8c82e1da97 100644 --- a/ydb/core/tx/datashard/datashard_write_operation.cpp +++ b/ydb/core/tx/datashard/datashard_write_operation.cpp @@ -100,6 +100,7 @@ std::tuple TValidatedWriteTxOperatio case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE: case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INSERT: case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPDATE: + case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT: break; default: return {NKikimrTxDataShard::TError::BAD_ARGUMENT, TStringBuilder() << OperationType << " operation is not supported now"}; @@ -162,6 +163,28 @@ std::tuple TValidatedWriteTxOperatio } } + if (OperationType == NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT) { + for (size_t i = tableInfo.KeyColumnIds.size(); i < ColumnIds.size(); i++) { // only data columns + auto* col = tableInfo.Columns.FindPtr(ColumnIds[i]); + if (col->IsKey) { + return {NKikimrTxDataShard::TError::BAD_ARGUMENT, TStringBuilder() << "Increment not allowed for key column " << ColumnIds[i]}; + } + auto type = col->Type.GetTypeId(); + switch (type) { + case NScheme::NTypeIds::Uint8: + case NScheme::NTypeIds::Int8: + case NScheme::NTypeIds::Uint16: + case NScheme::NTypeIds::Int16: + case NScheme::NTypeIds::Uint32: + case NScheme::NTypeIds::Int32: + case NScheme::NTypeIds::Uint64: + case NScheme::NTypeIds::Int64: + break; + default: + return {NKikimrTxDataShard::TError::BAD_ARGUMENT, TStringBuilder() << "Only integer types are supported by increment, but column " << ColumnIds[i] << " is not"}; + } + } + } TableId = TTableId(tableIdRecord.GetOwnerId(), tableIdRecord.GetTableId(), tableIdRecord.GetSchemaVersion()); SetTxKeys(tableInfo, tabletId, keyValidator); diff --git a/ydb/core/tx/datashard/execute_write_unit.cpp b/ydb/core/tx/datashard/execute_write_unit.cpp index a273bbdb5563..7a34f696a3f3 100644 --- a/ydb/core/tx/datashard/execute_write_unit.cpp +++ b/ydb/core/tx/datashard/execute_write_unit.cpp @@ -219,6 +219,11 @@ class TExecuteWriteUnit : public TExecutionUnit { userDb.UpdateRow(fullTableId, key, ops); break; } + case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT: { + FillOps(scheme, userTable, tableInfo, validatedOperation, rowIdx, ops); + userDb.IncrementRow(fullTableId, key, ops); + break; + } default: // Checked before in TWriteOperation Y_FAIL_S(operationType << " operation is not supported now"); @@ -231,7 +236,8 @@ class TExecuteWriteUnit : public TExecutionUnit { case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPSERT: case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_REPLACE: case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INSERT: - case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPDATE: { + case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPDATE: + case NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT: { DataShard.IncCounter(COUNTER_WRITE_ROWS, matrix.GetRowCount()); DataShard.IncCounter(COUNTER_WRITE_BYTES, matrix.GetBuffer().size()); break; diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp index c92746ff88d7..af72e2e6d89a 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.cpp @@ -2124,11 +2124,28 @@ void AddValueToCells(ui64 value, const TString& columnType, TVector& cell if (columnType == "Uint64") { cells.emplace_back(TCell((const char*)&value, sizeof(ui64))); } else if (columnType == "Uint32") { - ui32 value32 = (ui32)value; + ui32 value32 = static_cast(value); cells.emplace_back(TCell((const char*)&value32, sizeof(ui32))); + } else if (columnType == "Uint16") { + ui16 value16 = static_cast(value); + cells.emplace_back(TCell((const char*)&value16, sizeof(ui16))); + } else if (columnType == "Uint8") { + ui8 value8 = static_cast(value); + cells.emplace_back(TCell((const char*)&value8, sizeof(ui8))); + } else if (columnType == "Int64") { + i64 value64 = static_cast(value); + cells.push_back(TCell::Make(value64)); } else if (columnType == "Int32") { - i32 value32 = (i32)value; + i32 value32 = static_cast(value); cells.push_back(TCell::Make(value32)); + } else if (columnType == "Int16") { + i16 value16 = static_cast(value); + cells.push_back(TCell::Make(value16)); + } else if (columnType == "Int8") { + i8 value8 = static_cast(value); + cells.push_back(TCell::Make(value8)); + } else if (columnType == "Double") { + cells.emplace_back(TCell((const char*)&value, sizeof(double))); } else if (columnType == "Utf8") { stringValues.emplace_back(Sprintf("String_%" PRIu64, value)); cells.emplace_back(TCell(stringValues.back().c_str(), stringValues.back().size())); @@ -2266,6 +2283,25 @@ NKikimrDataEvents::TEvWriteResult Insert(TTestActorRuntime& runtime, TActorId se return Write(runtime, sender, shardId, std::move(request), expectedStatus); } +NKikimrDataEvents::TEvWriteResult Increment(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells) +{ + auto request = MakeWriteRequest(txId, txMode, NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT, tableId, columnIds, cells); + return Write(runtime, sender, shardId, std::move(request)); +} + +NKikimrDataEvents::TEvWriteResult Increment(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus) +{ + auto request = MakeWriteRequest(txId, txMode, NKikimrDataEvents::TEvWrite::TOperation::OPERATION_INCREMENT, tableId, columnIds, cells); + return Write(runtime, sender, shardId, std::move(request), expectedStatus); +} + +NKikimrDataEvents::TEvWriteResult Upsert(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells) +{ + auto request = MakeWriteRequest(txId, txMode, NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPSERT, tableId, columnIds, cells); + return Write(runtime, sender, shardId, std::move(request)); +} + + NKikimrDataEvents::TEvWriteResult Update(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, const TVector& columns, ui32 rowCount, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus) { auto request = MakeWriteRequest(txId, txMode, NKikimrDataEvents::TEvWrite::TOperation::OPERATION_UPDATE, tableId, columns, rowCount); diff --git a/ydb/core/tx/datashard/ut_common/datashard_ut_common.h b/ydb/core/tx/datashard/ut_common/datashard_ut_common.h index 61cb936c92b7..b5503ad85adc 100644 --- a/ydb/core/tx/datashard/ut_common/datashard_ut_common.h +++ b/ydb/core/tx/datashard/ut_common/datashard_ut_common.h @@ -805,6 +805,9 @@ NKikimrDataEvents::TEvWriteResult Replace(TTestActorRuntime& runtime, TActorId s NKikimrDataEvents::TEvWriteResult Delete(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, const TVector& columns, ui32 rowCount, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus = NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED); NKikimrDataEvents::TEvWriteResult Insert(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, const TVector& columns, ui32 rowCount, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus = NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED); NKikimrDataEvents::TEvWriteResult Update(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, const TVector& columns, ui32 rowCount, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus = NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED); +NKikimrDataEvents::TEvWriteResult Increment(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells); +NKikimrDataEvents::TEvWriteResult Increment(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus); +NKikimrDataEvents::TEvWriteResult Upsert(TTestActorRuntime& runtime, TActorId sender, ui64 shardId, const TTableId& tableId, std::optional txId, NKikimrDataEvents::TEvWrite::ETxMode txMode, const std::vector& columnIds, const std::vector& cells); NKikimrDataEvents::TEvWriteResult WaitForWriteCompleted(TTestActorRuntime& runtime, TActorId sender, NKikimrDataEvents::TEvWriteResult::EStatus expectedStatus = NKikimrDataEvents::TEvWriteResult::STATUS_COMPLETED); struct TEvWriteRow { diff --git a/ydb/core/tx/datashard/ya.make b/ydb/core/tx/datashard/ya.make index 57a118ff4590..44988b4826c5 100644 --- a/ydb/core/tx/datashard/ya.make +++ b/ydb/core/tx/datashard/ya.make @@ -283,6 +283,7 @@ PEERDIR( ydb/services/lib/sharding ydb/library/chunks_limiter yql/essentials/types/uuid + ydb/core/io_formats/cell_maker ) YQL_LAST_ABI_VERSION() From 3d2d4fba8cbb87ca64c9e4fc59d453b4f6bcf8d6 Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Thu, 19 Jun 2025 15:28:40 +0300 Subject: [PATCH 25/62] Rename column table (#16961) --- ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp | 17 +- ydb/core/protos/tx_columnshard.proto | 6 + ydb/core/tx/columnshard/columnshard_impl.cpp | 14 ++ ydb/core/tx/columnshard/columnshard_impl.h | 4 +- .../subscriber/abstract/events/event.h | 3 +- .../abstract/subscriber/subscriber.h | 7 +- .../subscriber/events/tx_completed/event.cpp | 9 + .../subscriber/events/tx_completed/event.h | 19 ++ .../subscriber/events/tx_completed/ya.make | 10 + .../tx/columnshard/subscriber/events/ya.make | 1 + ydb/core/tx/columnshard/tables_manager.cpp | 25 +++ ydb/core/tx/columnshard/tables_manager.h | 10 + .../test_helper/columnshard_ut_common.h | 10 + .../transactions/operators/schema.cpp | 136 +++++++++--- .../transactions/operators/schema.h | 6 +- .../transactions/tx_controller.cpp | 22 +- .../columnshard/transactions/tx_controller.h | 2 + .../ut_schema/ut_columnshard_move_table.cpp | 137 ++++++++++++ ydb/core/tx/columnshard/ut_schema/ya.make | 1 + .../tx/schemeshard/olap/manager/manager.h | 3 + .../schemeshard__operation_common.cpp | 66 ++++-- .../schemeshard__operation_common.h | 39 ++++ .../schemeshard__operation_move_table.cpp | 205 +++++++++++++----- .../schemeshard__operation_move_tables.cpp | 4 +- ydb/core/tx/schemeshard/schemeshard_impl.cpp | 24 ++ ydb/core/tx/schemeshard/schemeshard_impl.h | 6 + 26 files changed, 660 insertions(+), 126 deletions(-) create mode 100644 ydb/core/tx/columnshard/subscriber/events/tx_completed/event.cpp create mode 100644 ydb/core/tx/columnshard/subscriber/events/tx_completed/event.h create mode 100644 ydb/core/tx/columnshard/subscriber/events/tx_completed/ya.make create mode 100644 ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 5967585cb183..6f6e84ac6f0e 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -1156,20 +1156,21 @@ Y_UNIT_TEST_SUITE(KqpScheme) { CreateAndAlterTableWithPartitionSize(true); } - Y_UNIT_TEST(RenameTable) { + Y_UNIT_TEST_TWIN(RenameTable, СolumnTable) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); auto session = db.CreateSession().GetValueSync().GetSession(); { - TString query = R"( + TString query = ToString(R"( --!syntax_v1 CREATE TABLE `/Root/table` ( - Key Uint64, + Key Uint64 NOT NULL, Value String, PRIMARY KEY (Key) - ); - )"; + ) + )") + + (СolumnTable ? TString("WITH (STORE = COLUMN)") : ""); auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } @@ -1204,11 +1205,11 @@ Y_UNIT_TEST_SUITE(KqpScheme) { auto query = TStringBuilder() << R"( --!syntax_v1 CREATE TABLE `/Root/second` ( - Key Uint64, + Key Uint64 NOT NULL, Value String, PRIMARY KEY (Key) - ); - )"; + ) + )" + (СolumnTable ? TString("WITH (STORE = COLUMN)") : ""); auto result = session.ExecuteSchemeQuery(query).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto index 0ffad0993f03..0893d4cd3ca8 100644 --- a/ydb/core/protos/tx_columnshard.proto +++ b/ydb/core/protos/tx_columnshard.proto @@ -266,6 +266,11 @@ message TCreateTable { optional uint64 SchemaPresetVersionAdj = 6; } +message TMoveTable { + optional uint64 SrcPathId = 1; + optional uint64 DstPathId = 2; +} + message TInitShard { optional uint32 DataChannelCount = 1; optional uint64 OwnerPathId = 2; @@ -315,6 +320,7 @@ message TSchemaTxBody { TDropTable DropTable = 4; TAlterTable AlterTable = 5; TAlterStore AlterStore = 6; + TMoveTable MoveTable = 7; } optional NKikimrSchemeOp.TGranuleShardingInfo GranuleShardingInfo = 20; diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 179523763194..440c728c0310 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -253,6 +253,10 @@ void TColumnShard::RunSchemaTx(const NKikimrTxColumnShard::TSchemaTxBody& body, RunAlterStore(body.GetAlterStore(), version, txc); return; } + case NKikimrTxColumnShard::TSchemaTxBody::kMoveTable: { + RunMoveTable(body.GetMoveTable(), version, txc); + return; + } case NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET: { break; } @@ -400,6 +404,16 @@ void TColumnShard::RunDropTable(const NKikimrTxColumnShard::TDropTable& dropProt TablesManager.DropTable(*internalPathId, version, db); } +void TColumnShard::RunMoveTable(const NKikimrTxColumnShard::TMoveTable& proto, const NOlap::TSnapshot& /*version*/, + NTabletFlatExecutor::TTransactionContext& txc) { + NIceDb::TNiceDb db(txc.DB); + + const auto srcPathId = TSchemeShardLocalPathId::FromRawValue(proto.GetSrcPathId()); + const auto dstPathId = TSchemeShardLocalPathId::FromRawValue(proto.GetDstPathId()); + TablesManager.MoveTableProgress(db, srcPathId, dstPathId); +} + + void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) { NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 0dcda3a5b224..f981afb7d3c7 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -179,7 +179,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa friend class TTxMonitoring; friend class TTxRemoveSharedBlobs; friend class TTxFinishAsyncTransaction; - friend class TWaitEraseTablesTxSubscriber; + friend class TWaitOnProposeTxSubscriberBase; friend class TTxPersistSubDomainOutOfSpace; friend class TTxPersistSubDomainPathId; friend class TSpaceWatcher; @@ -562,6 +562,8 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa const NKikimrTxColumnShard::TDropTable& body, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc); void RunAlterStore( const NKikimrTxColumnShard::TAlterStore& body, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc); + void RunMoveTable( + const NKikimrTxColumnShard::TMoveTable& proto, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc); void SetupCompaction(const std::set& pathIds); void StartCompaction(const std::shared_ptr& guard); diff --git a/ydb/core/tx/columnshard/subscriber/abstract/events/event.h b/ydb/core/tx/columnshard/subscriber/abstract/events/event.h index 0ebea7af609f..3f58d6c0748a 100644 --- a/ydb/core/tx/columnshard/subscriber/abstract/events/event.h +++ b/ydb/core/tx/columnshard/subscriber/abstract/events/event.h @@ -6,7 +6,8 @@ namespace NKikimr::NColumnShard::NSubscriber { enum class EEventType { Undefined, - TablesErased + TablesErased, + TxCompleted, }; class ISubscriptionEvent { diff --git a/ydb/core/tx/columnshard/subscriber/abstract/subscriber/subscriber.h b/ydb/core/tx/columnshard/subscriber/abstract/subscriber/subscriber.h index b3e2ae6123af..e9bba1210d6e 100644 --- a/ydb/core/tx/columnshard/subscriber/abstract/subscriber/subscriber.h +++ b/ydb/core/tx/columnshard/subscriber/abstract/subscriber/subscriber.h @@ -10,13 +10,8 @@ class TColumnShard; namespace NKikimr::NColumnShard::NSubscriber { class ISubscriber { -private: - virtual bool DoOnEvent(const std::shared_ptr& ev, TColumnShard& shard) = 0; public: - bool OnEvent(const std::shared_ptr& ev, TColumnShard& shard) { - return DoOnEvent(ev, shard); - } - + virtual void OnEvent(const std::shared_ptr& ev, TColumnShard& shard) = 0; virtual std::set GetEventTypes() const = 0; virtual bool IsFinished() const = 0; diff --git a/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.cpp b/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.cpp new file mode 100644 index 000000000000..72828821181f --- /dev/null +++ b/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.cpp @@ -0,0 +1,9 @@ +#include "event.h" + +namespace NKikimr::NColumnShard::NSubscriber { + +TString TEventTxCompleted::DoDebugString() const { + return "tx_id=" + std::to_string(TxId); +} + +} //namespace NKikimr::NColumnShard::NSubscriber diff --git a/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.h b/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.h new file mode 100644 index 000000000000..897e01f6603e --- /dev/null +++ b/ydb/core/tx/columnshard/subscriber/events/tx_completed/event.h @@ -0,0 +1,19 @@ +#pragma once +#include + +namespace NKikimr::NColumnShard::NSubscriber { + +class TEventTxCompleted: public ISubscriptionEvent { +private: + using TBase = ISubscriptionEvent; + YDB_READONLY_DEF(ui64, TxId); + virtual TString DoDebugString() const override; +public: + TEventTxCompleted(const ui64 txId) + : TBase(EEventType::TxCompleted) + , TxId(txId) + { + } +}; + +} //namespace NKikimr::NColumnShard::NSubscriber \ No newline at end of file diff --git a/ydb/core/tx/columnshard/subscriber/events/tx_completed/ya.make b/ydb/core/tx/columnshard/subscriber/events/tx_completed/ya.make new file mode 100644 index 000000000000..9d6bfdded2da --- /dev/null +++ b/ydb/core/tx/columnshard/subscriber/events/tx_completed/ya.make @@ -0,0 +1,10 @@ +LIBRARY() + +SRCS( + event.cpp +) + +PEERDIR( +) + +END() diff --git a/ydb/core/tx/columnshard/subscriber/events/ya.make b/ydb/core/tx/columnshard/subscriber/events/ya.make index 5dbbd1f69e96..efca1042868d 100644 --- a/ydb/core/tx/columnshard/subscriber/events/ya.make +++ b/ydb/core/tx/columnshard/subscriber/events/ya.make @@ -2,6 +2,7 @@ LIBRARY() PEERDIR( ydb/core/tx/columnshard/subscriber/events/tables_erased + ydb/core/tx/columnshard/subscriber/events/tx_completed ) END() diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index c1f33ce06e7b..1c8e3383f5d7 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -430,4 +430,29 @@ bool TTablesManager::TryFinalizeDropPathOnComplete(const TInternalPathId pathId) return true; } +void TTablesManager::MoveTablePropose(const TSchemeShardLocalPathId schemeShardLocalPathId) { + NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("scheme_shard_local_path_id", schemeShardLocalPathId); + const auto& internalPathId = ResolveInternalPathId(schemeShardLocalPathId); + AFL_VERIFY(internalPathId); + AFL_VERIFY(RenamingLocalToInternal.emplace(schemeShardLocalPathId, *internalPathId).second)("internal_path_id", internalPathId); + AFL_VERIFY(SchemeShardLocalToInternal.erase(schemeShardLocalPathId)); +} + +void TTablesManager::MoveTableProgress(NIceDb::TNiceDb& db, const TSchemeShardLocalPathId oldSchemeShardLocalPathId, const TSchemeShardLocalPathId newSchemeShardLocalPathId) { + NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD) + ("event", "move_table_progress")("old_path_id", oldSchemeShardLocalPathId)("new_path_id", newSchemeShardLocalPathId); + AFL_VERIFY(!ResolveInternalPathId(oldSchemeShardLocalPathId)); + AFL_VERIFY(!ResolveInternalPathId(newSchemeShardLocalPathId)); + const auto* internalPathId = RenamingLocalToInternal.FindPtr(oldSchemeShardLocalPathId); + AFL_VERIFY(internalPathId); + AFL_VERIFY(HasTable(*internalPathId)); + auto* table = Tables.FindPtr(*internalPathId); + AFL_VERIFY(table); + table->UpdateLocalPathId(db, newSchemeShardLocalPathId); + AFL_VERIFY(RenamingLocalToInternal.erase(oldSchemeShardLocalPathId)); + AFL_VERIFY(SchemeShardLocalToInternal.emplace(newSchemeShardLocalPathId, *internalPathId).second); + NYDBTest::TControllers::GetColumnShardController()->OnDeletePathId(TabletId, {*internalPathId, oldSchemeShardLocalPathId}); + NYDBTest::TControllers::GetColumnShardController()->OnAddPathId(TabletId, table->GetPathId()); +} + } // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 79a9ced3945a..86b5e90fc9c1 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -121,6 +121,11 @@ class TTableInfo { Versions.insert(snapshot); } + void UpdateLocalPathId(NIceDb::TNiceDb& db, const TSchemeShardLocalPathId newPathId) { + Schema::SaveTableSchemeShardLocalPathId(db, InternalPathId, newPathId); + SchemeShardLocalPathId = newPathId; + } + bool IsDropped(const std::optional& minReadSnapshot = std::nullopt) const { if (!DropVersion) { return false; @@ -199,6 +204,7 @@ class TTablesManager: public NOlap::IPathIdTranslator { private: THashMap Tables; THashMap SchemeShardLocalToInternal; + THashMap RenamingLocalToInternal; // Paths that are being renamed THashSet SchemaPresetsIds; THashMap ActualSchemaForPreset; std::map> PathsToDrop; @@ -275,6 +281,10 @@ class TTablesManager: public NOlap::IPathIdTranslator { return !!PrimaryIndex; } + void MoveTablePropose(const TSchemeShardLocalPathId schemeShardLocalPathId); + void MoveTableProgress(NIceDb::TNiceDb& db, const TSchemeShardLocalPathId oldSchemeShardLocalPathId, const TSchemeShardLocalPathId newSchemeShardLocalPathId); + + NOlap::IColumnEngine& MutablePrimaryIndex() { Y_ABORT_UNLESS(!!PrimaryIndex); return *PrimaryIndex; diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 57462809f48f..38f878b400ed 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -382,6 +382,16 @@ struct TTestSchema { return out; } + static TString MoveTableTxBody(ui64 srcPathId, ui64 dstPathId, ui32 version) { + NKikimrTxColumnShard::TSchemaTxBody tx; + tx.MutableMoveTable()->SetSrcPathId(srcPathId); + tx.MutableMoveTable()->SetDstPathId(dstPathId); + tx.MutableSeqNo()->SetRound(version); + TString out; + Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&out); + return out; + } + static THashMap BuildSnapshot(const TTableSpecials& specials); static TString CommitTxBody(ui64, const std::vector& writeIds) { diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.cpp b/ydb/core/tx/columnshard/transactions/operators/schema.cpp index 38726fb0e29f..53446e2d9b4c 100644 --- a/ydb/core/tx/columnshard/transactions/operators/schema.cpp +++ b/ydb/core/tx/columnshard/transactions/operators/schema.cpp @@ -1,46 +1,88 @@ #include "schema.h" #include #include +#include #include +#include #include +#include namespace NKikimr::NColumnShard { -class TWaitEraseTablesTxSubscriber: public NSubscriber::ISubscriber { +class TWaitOnProposeTxSubscriberBase : public NSubscriber::ISubscriber { + const ui64 TxId; +protected: + TWaitOnProposeTxSubscriberBase(const ui64 txId) + : TxId(txId) { } + + void OnEvent(const std::shared_ptr& ev, TColumnShard& shard) { + const NActors::TLogContextGuard g = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "on_subscriber_event")("tx_id", TxId)("event", ev->DebugString()); + AFL_VERIFY(!IsFinished()); + DoOnEvent(ev); + if (IsFinished()) { + shard.Execute(new TTxFinishAsyncTransaction(shard, TxId)); + } + } + +protected: + virtual void DoOnEvent(const std::shared_ptr& ev) = 0; +}; + + +class TWaitEraseTablesTxSubscriber: public TWaitOnProposeTxSubscriberBase { private: THashSet WaitTables; - const ui64 TxId; public: + TWaitEraseTablesTxSubscriber(const ui64 txId, THashSet&& waitTables) + : TWaitOnProposeTxSubscriberBase(txId) + , WaitTables(std::move(waitTables)) { + } + virtual std::set GetEventTypes() const override { return { NSubscriber::EEventType::TablesErased }; } - virtual bool DoOnEvent(const std::shared_ptr& ev, TColumnShard& shard) override { + virtual void DoOnEvent(const std::shared_ptr& ev) override { AFL_VERIFY(ev->GetType() == NSubscriber::EEventType::TablesErased); auto* evErased = static_cast(ev.get()); - bool result = false; for (auto&& i : evErased->GetPathIds()) { - result |= WaitTables.erase(i); + WaitTables.erase(i); } - AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("event", "on_event")("remained", JoinSeq(",", WaitTables)); - if (WaitTables.empty()) { - shard.Execute(new TTxFinishAsyncTransaction(shard, TxId)); - } - return result; + AFL_NOTICE(NKikimrServices::TX_COLUMNSHARD)("remained", JoinSeq(",", WaitTables)); } virtual bool IsFinished() const override { return WaitTables.empty(); } +}; + +class TWaitTxs: public TWaitOnProposeTxSubscriberBase { + THashSet TxIdsToWait; +public: + TWaitTxs(const ui64 txId, const THashSet&& txIdsToWait) + : TWaitOnProposeTxSubscriberBase(txId) + , TxIdsToWait(std::move(txIdsToWait)) { + } + std::set GetEventTypes() const override { + return { NSubscriber::EEventType::TxCompleted }; + } - TWaitEraseTablesTxSubscriber(const THashSet& waitTables, const ui64 txId) - : WaitTables(waitTables) - , TxId(txId) { + bool IsFinished() const override { + return TxIdsToWait.empty(); + } + virtual void DoOnEvent(const std::shared_ptr& ev) override { + AFL_VERIFY(ev->GetType() == NSubscriber::EEventType::TxCompleted); + const auto* evCompleted = static_cast(ev.get()); + AFL_VERIFY(TxIdsToWait.erase(evCompleted->GetTxId())); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("completed", evCompleted->GetTxId())("remained", JoinSeq(",", TxIdsToWait)); } }; + + TTxController::TProposeResult TSchemaTransactionOperator::DoStartProposeOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) { + AFL_VERIFY(!WaitOnPropose); auto seqNo = SeqNoFromProto(SchemaTxBody.GetSeqNo()); auto lastSeqNo = owner.LastSchemaSeqNo; @@ -62,24 +104,55 @@ TTxController::TProposeResult TSchemaTransactionOperator::DoStartProposeOnExecut if (validationStatus.IsFail()) { return TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "Invalid schema: " + validationStatus.GetErrorMessage()); } - WaitPathIdsToErase = GetNotErasedTableIds(owner, SchemaTxBody.GetInitShard().GetTables()); + auto pathIdsToErase = GetNotErasedTableIds(owner, SchemaTxBody.GetInitShard().GetTables()); + if (!pathIdsToErase.empty()) { + WaitOnPropose = std::make_shared(GetTxId(), std::move(pathIdsToErase)); + } } break; case NKikimrTxColumnShard::TSchemaTxBody::kEnsureTables: { - auto validationStatus = ValidateTables(SchemaTxBody.GetEnsureTables().GetTables()); + const auto& tables = SchemaTxBody.GetEnsureTables().GetTables(); + auto validationStatus = ValidateTables(tables); if (validationStatus.IsFail()) { return TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "Invalid schema: " + validationStatus.GetErrorMessage()); } - WaitPathIdsToErase = GetNotErasedTableIds(owner, SchemaTxBody.GetEnsureTables().GetTables()); + auto waitPathIdsToErase = GetNotErasedTableIds(owner, SchemaTxBody.GetEnsureTables().GetTables()); + if (!waitPathIdsToErase.empty()) { + WaitOnPropose = std::make_shared(GetTxId(), std::move(waitPathIdsToErase)); + } } break; case NKikimrTxColumnShard::TSchemaTxBody::kAlterTable: case NKikimrTxColumnShard::TSchemaTxBody::kAlterStore: case NKikimrTxColumnShard::TSchemaTxBody::kDropTable: + break; + case NKikimrTxColumnShard::TSchemaTxBody::kMoveTable: + { + const auto srcSchemeShardLocalPathId = TSchemeShardLocalPathId::FromRawValue(SchemaTxBody.GetMoveTable().GetSrcPathId()); + const auto dstSchemeShardLocalPathId = TSchemeShardLocalPathId::FromRawValue(SchemaTxBody.GetMoveTable().GetDstPathId()); + AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("propose_execute", "move_table")("src", srcSchemeShardLocalPathId)("dst", dstSchemeShardLocalPathId); + if (!owner.TablesManager.ResolveInternalPathId(srcSchemeShardLocalPathId)) { + return TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "No such table"); + } + if (owner.TablesManager.ResolveInternalPathId(dstSchemeShardLocalPathId)) { + return TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "Rename to existing table"); + } + auto txIdsToWait = owner.GetProgressTxController().GetTxs(); //TODO #8650 Get transaction for moving pathId only + if (!txIdsToWait.empty()) { + //TODO persist txIdsToWait to recover on table init + AFL_VERIFY(!txIdsToWait.contains(GetTxId()))("tx_id", GetTxId())("tx_ids", JoinSeq(",", txIdsToWait)); + WaitOnPropose = std::make_shared(GetTxId(), std::move(txIdsToWait)); + } + owner.TablesManager.MoveTablePropose(srcSchemeShardLocalPathId); + break; + } case NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET: break; } + if (WaitOnPropose) { + owner.Subscribers->RegisterSubscriber(WaitOnPropose); + } owner.UpdateSchemaSeqNo(seqNo, txc); return TProposeResult(); @@ -164,40 +237,53 @@ NKikimr::TConclusionStatus TSchemaTransactionOperator::ValidateTables(::google:: } void TSchemaTransactionOperator::DoOnTabletInit(TColumnShard& owner) { - AFL_VERIFY(WaitPathIdsToErase.empty()); + AFL_VERIFY(!WaitOnPropose); switch (SchemaTxBody.TxBody_case()) { case NKikimrTxColumnShard::TSchemaTxBody::kInitShard: break; case NKikimrTxColumnShard::TSchemaTxBody::kEnsureTables: { + THashSet waitPathIdsToErase; for (auto&& i : SchemaTxBody.GetEnsureTables().GetTables()) { const auto& schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(i); if (const auto internalPathId = owner.TablesManager.ResolveInternalPathId(schemeShardLocalPathId)) { if (owner.TablesManager.HasTable(*internalPathId, true)) { - WaitPathIdsToErase.emplace(*internalPathId); + waitPathIdsToErase.emplace(*internalPathId); } } } + if (!waitPathIdsToErase.empty()) { + WaitOnPropose = std::make_shared(GetTxId(), std::move(waitPathIdsToErase)); + } } break; case NKikimrTxColumnShard::TSchemaTxBody::kAlterTable: case NKikimrTxColumnShard::TSchemaTxBody::kAlterStore: case NKikimrTxColumnShard::TSchemaTxBody::kDropTable: + break; + case NKikimrTxColumnShard::TSchemaTxBody::kMoveTable: + { + const auto srcSchemeShardLocalPathId = TSchemeShardLocalPathId::FromRawValue(SchemaTxBody.GetMoveTable().GetSrcPathId()); + const auto dstSchemeShardLocalPathId = TSchemeShardLocalPathId::FromRawValue(SchemaTxBody.GetMoveTable().GetDstPathId()); + + AFL_VERIFY(owner.TablesManager.ResolveInternalPathId(srcSchemeShardLocalPathId)); + AFL_VERIFY(!owner.TablesManager.ResolveInternalPathId(dstSchemeShardLocalPathId)); + //TODO recover txIdsToWait + //WaitOnPropose = std::make_shared(GetTxId(), txIds); + } case NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET: break; } - if (WaitPathIdsToErase.size()) { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "wait_remove_path_id")("pathes", JoinSeq(",", WaitPathIdsToErase))("tx_id", GetTxId()); - owner.Subscribers->RegisterSubscriber(std::make_shared(WaitPathIdsToErase, GetTxId())); + if (WaitOnPropose) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "wait_on_propose")("tx_id", GetTxId()); + owner.Subscribers->RegisterSubscriber(WaitOnPropose); } else { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "remove_pathes_cleaned")("tx_id", GetTxId()); owner.Execute(new TTxFinishAsyncTransaction(owner, GetTxId())); } } -void TSchemaTransactionOperator::DoStartProposeOnComplete(TColumnShard& owner, const TActorContext& /*ctx*/) { - AFL_VERIFY(WaitPathIdsToErase.size()); - owner.Subscribers->RegisterSubscriber(std::make_shared(WaitPathIdsToErase, GetTxId())); +void TSchemaTransactionOperator::DoStartProposeOnComplete(TColumnShard& /*owner*/, const TActorContext& /*ctx*/) { } -} +} //namespace NKikimr::NColumnShard \ No newline at end of file diff --git a/ydb/core/tx/columnshard/transactions/operators/schema.h b/ydb/core/tx/columnshard/transactions/operators/schema.h index 07d1f6e2f917..5983d545bffc 100644 --- a/ydb/core/tx/columnshard/transactions/operators/schema.h +++ b/ydb/core/tx/columnshard/transactions/operators/schema.h @@ -17,7 +17,7 @@ class TSchemaTransactionOperator: public IProposeTxOperator, public TMonitoringO std::unique_ptr TxAddSharding; NKikimrTxColumnShard::TSchemaTxBody SchemaTxBody; THashSet NotifySubscribers; - THashSet WaitPathIdsToErase; + std::shared_ptr WaitOnPropose; virtual void DoOnTabletInit(TColumnShard& owner) override; @@ -58,12 +58,14 @@ class TSchemaTransactionOperator: public IProposeTxOperator, public TMonitoringO return "Scheme:AlterStore"; case NKikimrTxColumnShard::TSchemaTxBody::kDropTable: return "Scheme:DropTable"; + case NKikimrTxColumnShard::TSchemaTxBody::kMoveTable: + return "Scheme:MoveTable"; case NKikimrTxColumnShard::TSchemaTxBody::TXBODY_NOT_SET: return "Scheme:TXBODY_NOT_SET"; } } virtual bool DoIsAsync() const override { - return WaitPathIdsToErase.size(); + return !!WaitOnPropose; } virtual bool DoParse(TColumnShard& owner, const TString& data) override { if (!SchemaTxBody.ParseFromString(data)) { diff --git a/ydb/core/tx/columnshard/transactions/tx_controller.cpp b/ydb/core/tx/columnshard/transactions/tx_controller.cpp index c423e8915e3a..c0561b198ade 100644 --- a/ydb/core/tx/columnshard/transactions/tx_controller.cpp +++ b/ydb/core/tx/columnshard/transactions/tx_controller.cpp @@ -3,6 +3,7 @@ #include "transactions/tx_finish_async.h" #include +#include namespace NKikimr::NColumnShard { @@ -145,7 +146,7 @@ bool TTxController::AbortTx(const TPlanQueueItem planQueueItem, NTabletFlatExecu Counters.OnAbortTx(opIt->second->GetOpType()); AFL_WARN(NKikimrServices::TX_COLUMNSHARD_TX)("event", "abort_tx")("tx_id", planQueueItem.TxId); - AFL_VERIFY(Operators.erase(planQueueItem.TxId))("tx_id", planQueueItem.TxId); + OnTxCompleted(planQueueItem.TxId); AFL_VERIFY(DeadlineQueue.erase(planQueueItem))("tx_id", planQueueItem.TxId); NIceDb::TNiceDb db(txc.DB); Schema::EraseTxInfo(db, planQueueItem.TxId); @@ -167,7 +168,7 @@ bool TTxController::CompleteOnCancel(const ui64 txId, const TActorContext& ctx) DeadlineQueue.erase(TPlanQueueItem(opIt->second->GetTxInfo().MaxStep, txId)); } AFL_WARN(NKikimrServices::TX_COLUMNSHARD_TX)("event", "cancel_tx")("tx_id", txId); - Operators.erase(txId); + OnTxCompleted(txId); return true; } @@ -212,7 +213,7 @@ void TTxController::ProgressOnExecute(const ui64 txId, NTabletFlatExecutor::TTra AFL_VERIFY(opIt != Operators.end())("tx_id", txId); Counters.OnFinishPlannedTx(opIt->second->GetOpType()); AFL_WARN(NKikimrServices::TX_COLUMNSHARD_TX)("event", "finished_tx")("tx_id", txId); - AFL_VERIFY(Operators.erase(txId)); + OnTxCompleted(txId); Schema::EraseTxInfo(db, txId); } @@ -220,6 +221,19 @@ void TTxController::ProgressOnComplete(const TPlanQueueItem& txItem) { AFL_VERIFY(RunningQueue.erase(txItem))("info", txItem.DebugString()); } +void TTxController::OnTxCompleted(const ui64 txId) { + AFL_VERIFY(Operators.erase(txId)); + Owner.Subscribers->OnEvent(std::make_shared(txId)); +} + +THashSet TTxController::GetTxs() const { + THashSet result; + for (const auto& [txId, _]: Operators) { + result.emplace(txId); + } + return result; +} + std::optional TTxController::GetPlannedTx() const { if (PlanQueue.empty()) { return std::nullopt; @@ -395,7 +409,7 @@ void TTxController::FinishProposeOnComplete(const ui64 txId, const TActorContext void TTxController::ITransactionOperator::SwitchStateVerified(const EStatus from, const EStatus to) { AFL_VERIFY(!Status || *Status == from)("error", "incorrect expected status")("real_state", *Status)("expected", from)( - "details", DebugString()); + "details", DebugString()); Status = to; } diff --git a/ydb/core/tx/columnshard/transactions/tx_controller.h b/ydb/core/tx/columnshard/transactions/tx_controller.h index 5b9c02f9e83f..ec60a45f9dd0 100644 --- a/ydb/core/tx/columnshard/transactions/tx_controller.h +++ b/ydb/core/tx/columnshard/transactions/tx_controller.h @@ -426,6 +426,7 @@ class TTxController { TTxInfo RegisterTxWithDeadline(const std::shared_ptr& txOperator, const TString& txBody, NTabletFlatExecutor::TTransactionContext& txc); bool StartedFlag = false; + void OnTxCompleted(const ui64 txId); public: TTxController(TColumnShard& owner); @@ -488,6 +489,7 @@ class TTxController { std::optional PopFirstPlannedTx(); void ProgressOnExecute(const ui64 txId, NTabletFlatExecutor::TTransactionContext& txc); void ProgressOnComplete(const TPlanQueueItem& tx); + THashSet GetTxs() const; //TODO #8650 GetTxsByPathId std::optional GetPlannedTx() const; TPlanQueueItem GetFrontTx() const; diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp new file mode 100644 index 000000000000..472ee3e76913 --- /dev/null +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_move_table.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace NKikimr { + +using namespace NColumnShard; +using namespace Tests; +using namespace NTxUT; + +using TTypeId = NScheme::TTypeId; +using TTypeInfo = NScheme::TTypeInfo; +using TDefaultTestsController = NKikimr::NYDBTest::NColumnShard::TController; + + +Y_UNIT_TEST_SUITE(MoveTable) { + Y_UNIT_TEST(EmptyTable) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + TActorId sender = runtime.AllocateEdgeActor(); + + const ui64 srcPathId = 1; + TestTableDescription testTabe{}; + auto planStep = PrepareTablet(runtime, srcPathId, testTabe.Schema); + + ui64 txId = 10; + const ui64 dstPathId = 2; + planStep = ProposeSchemaTx(runtime, sender, TTestSchema::MoveTableTxBody(srcPathId, dstPathId, 1), ++txId); + PlanSchemaTx(runtime, sender, {planStep, txId}); + } + + Y_UNIT_TEST(WithUncomittedData) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + TActorId sender = runtime.AllocateEdgeActor(); + + const ui64 srcPathId = 1; + TestTableDescription testTabe{}; + auto planStep = PrepareTablet(runtime, srcPathId, testTabe.Schema); + + ui64 txId = 10; + int writeId = 10; + std::vector writeIds; + const bool ok = WriteData(runtime, sender, writeId++, srcPathId, MakeTestBlob({0, 100}, testTabe.Schema), testTabe.Schema, true, &writeIds); + UNIT_ASSERT(ok); + const ui64 dstPathId = 2; + planStep = ProposeSchemaTx(runtime, sender, TTestSchema::MoveTableTxBody(srcPathId, dstPathId, 1), ++txId); + PlanSchemaTx(runtime, sender, {planStep, txId}); + } + + Y_UNIT_TEST(WithData) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + TActorId sender = runtime.AllocateEdgeActor(); + + const ui64 srcPathId = 1; + TestTableDescription testTabe{}; + auto planStep = PrepareTablet(runtime, srcPathId, testTabe.Schema); + + ui64 txId = 10; + int writeId = 10; + std::vector writeIds; + const bool ok = WriteData(runtime, sender, writeId++, srcPathId, MakeTestBlob({0, 100}, testTabe.Schema), testTabe.Schema, true, &writeIds); + UNIT_ASSERT(ok); + planStep = ProposeCommit(runtime, sender, ++txId, writeIds); + PlanCommit(runtime, sender, planStep, txId); + + const ui64 dstPathId = 2; + planStep = ProposeSchemaTx(runtime, sender, TTestSchema::MoveTableTxBody(srcPathId, dstPathId, 1), ++txId); + PlanSchemaTx(runtime, sender, {planStep, txId}); + + { + TShardReader reader(runtime, TTestTxConfig::TxTablet0, dstPathId, NOlap::TSnapshot(planStep, txId)); + auto rb = reader.ReadAll(); + UNIT_ASSERT(rb); + UNIT_ASSERT_EQUAL(rb->num_rows(), 100); + } + + { + TShardReader reader(runtime, TTestTxConfig::TxTablet0, srcPathId, NOlap::TSnapshot(planStep, txId)); + auto rb = reader.ReadAll(); + UNIT_ASSERT(!rb); + } + + } + + Y_UNIT_TEST(RenameAbsentTable_Negative) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + TActorId sender = runtime.AllocateEdgeActor(); + + const ui64 srcPathId = 1; + TestTableDescription testTabe{}; + const auto& planStep = PrepareTablet(runtime, srcPathId, testTabe.Schema); + Y_UNUSED(planStep); + + const ui64 absentPathId = 111; + const ui64 dstPathId = 2; + ui64 txId = 10; + ProposeSchemaTxFail(runtime, sender, TTestSchema::MoveTableTxBody(absentPathId, dstPathId, 1), ++txId); + } + + Y_UNIT_TEST(RenameToItself_Negative) { + TTestBasicRuntime runtime; + TTester::Setup(runtime); + auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + TActorId sender = runtime.AllocateEdgeActor(); + + const ui64 srcPathId = 1; + TestTableDescription testTabe{}; + const auto& planStep = PrepareTablet(runtime, srcPathId, testTabe.Schema); + Y_UNUSED(planStep); + ui64 txId = 10; + ProposeSchemaTxFail(runtime, sender, TTestSchema::MoveTableTxBody(srcPathId, srcPathId, 1), ++txId); + } +} +}// namespace NKikimr \ No newline at end of file diff --git a/ydb/core/tx/columnshard/ut_schema/ya.make b/ydb/core/tx/columnshard/ut_schema/ya.make index 5f3492047ce4..281a19b13012 100644 --- a/ydb/core/tx/columnshard/ut_schema/ya.make +++ b/ydb/core/tx/columnshard/ut_schema/ya.make @@ -30,6 +30,7 @@ YQL_LAST_ABI_VERSION() SRCS( ut_columnshard_schema.cpp + ut_columnshard_move_table.cpp ) END() diff --git a/ydb/core/tx/schemeshard/olap/manager/manager.h b/ydb/core/tx/schemeshard/olap/manager/manager.h index a2697cf5b593..129c544e594c 100644 --- a/ydb/core/tx/schemeshard/olap/manager/manager.h +++ b/ydb/core/tx/schemeshard/olap/manager/manager.h @@ -40,6 +40,9 @@ class TTablesStorage { Y_DEBUG_ABORT_UNLESS(Object); return *Object; } + TColumnTableInfo::TPtr GetPtr() const { + return Object; + } }; class TTableCreateOperator: public TTableReadGuard { diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp index fb3d928170bf..538c45284021 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.cpp @@ -593,43 +593,46 @@ bool CollectProposeTransactionResults( return CollectProposeTxResults(ev, operationId, context, prepared, toString); } -bool CollectSchemaChanged( +namespace { + +template +bool CollectSchemaChangedImpl( const TOperationId& operationId, - const TEvDataShard::TEvSchemaChanged::TPtr& ev, + const TEvent& ev, TOperationContext& context) { auto ssId = context.SS->SelfTabletId(); const auto& evRecord = ev->Get()->Record; - const TActorId ackTo = ev->Get()->GetSource(); + const TActorId ackTo = TEvSchemaChangedTraits::GetSource(ev); - auto datashardId = TTabletId(evRecord.GetOrigin()); + auto shardId = TTabletId(evRecord.GetOrigin()); Y_ABORT_UNLESS(context.SS->FindTx(operationId)); TTxState& txState = *context.SS->FindTx(operationId); - auto shardIdx = context.SS->MustGetShardIdx(datashardId); + auto shardIdx = context.SS->MustGetShardIdx(shardId); Y_ABORT_UNLESS(context.SS->ShardInfos.contains(shardIdx)); // Save this notification if was received earlier than the Tx switched to ProposedWaitParts state - ui32 generation = evRecord.GetGeneration(); + const auto& generation = TEvSchemaChangedTraits::GetGeneration(ev); auto pTablet = txState.SchemeChangeNotificationReceived.FindPtr(shardIdx); - if (pTablet && pTablet->second >= generation) { + if (pTablet && generation && (pTablet->second >= *generation)) { LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "CollectSchemaChanged Ignore TEvDataShard::TEvSchemaChanged as outdated" + "CollectSchemaChanged Ignore " << TEvSchemaChangedTraits::GetName() << " as outdated" << ", operationId: " << operationId << ", shardIdx: " << shardIdx - << ", datashard " << datashardId + << ", shard " << shardId << ", event generation: " << generation << ", known generation: " << pTablet->second << ", at schemeshard: " << ssId); return false; } - txState.SchemeChangeNotificationReceived[shardIdx] = std::make_pair(ackTo, generation); + txState.SchemeChangeNotificationReceived[shardIdx] = std::make_pair(ackTo, generation ? *generation : 0); - if (evRecord.HasOpResult()) { + if (TEvSchemaChangedTraits::HasOpResult(ev)) { // TODO: remove TxBackup handling Y_DEBUG_ABORT_UNLESS(txState.TxType == TTxState::TxBackup || txState.TxType == TTxState::TxRestore); } @@ -646,10 +649,10 @@ bool CollectSchemaChanged( txState.ShardsInProgress.erase(shardIdx); LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - "CollectSchemaChanged accept TEvDataShard::TEvSchemaChanged" + "CollectSchemaChanged accept " << TEvSchemaChangedTraits::GetName() << ", operationId: " << operationId << ", shardIdx: " << shardIdx - << ", datashard: " << datashardId + << ", shard: " << shardId << ", left await: " << txState.ShardsInProgress.size() << ", txState.State: " << TTxState::StateName(txState.State) << ", txState.ReadyForNotifications: " << txState.ReadyForNotifications @@ -666,6 +669,24 @@ bool CollectSchemaChanged( return false; } +} //namespace + +bool CollectSchemaChanged( + const TOperationId& operationId, + const TEvDataShard::TEvSchemaChanged::TPtr& ev, + TOperationContext& context) +{ + return CollectSchemaChangedImpl<>(operationId, ev, context); +} + +bool CollectSchemaChanged( + const TOperationId& operationId, + const TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, + TOperationContext& context) +{ + return CollectSchemaChangedImpl(operationId, ev, context); +} + void AckAllSchemaChanges(const TOperationId &operationId, TTxState &txState, TOperationContext &context) { TTabletId ssId = context.SS->SelfTabletId(); @@ -1005,21 +1026,22 @@ TProposedWaitParts::TProposedWaitParts(TOperationId id, TTxState::ETxState nextS ); } -bool TProposedWaitParts::HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) { +template +bool TProposedWaitParts::HandleReplyImpl(const TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); const auto& evRecord = ev->Get()->Record; LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " at tablet: " << ssId); LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " at tablet: " << ssId << " message: " << evRecord.ShortDebugString()); if (!CollectSchemaChanged(OperationId, ev, context)) { LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " CollectSchemaChanged: false"); return false; } @@ -1029,7 +1051,7 @@ bool TProposedWaitParts::HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, T if (!txState.ReadyForNotifications) { LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " ReadyForNotifications: false"); return false; } @@ -1037,6 +1059,14 @@ bool TProposedWaitParts::HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, T return true; } +bool TProposedWaitParts::HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) { + return HandleReplyImpl(ev, context); +} + +bool TProposedWaitParts::HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) { + return HandleReplyImpl(ev, context); +} + bool TProposedWaitParts::ProgressState(TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_common.h b/ydb/core/tx/schemeshard/schemeshard__operation_common.h index 9b41a0ffce86..5a378dfeb239 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_common.h +++ b/ydb/core/tx/schemeshard/schemeshard__operation_common.h @@ -15,6 +15,41 @@ namespace NKikimr::NSchemeShard { class TSchemeShard; +template +struct TEvSchemaChangedTraits; + +template<> +struct TEvSchemaChangedTraits { + static TActorId GetSource(const TEvDataShard::TEvSchemaChanged::TPtr& ev) { + return TActorId{ev->Get()->GetSource()}; + } + static std::optional GetGeneration(const TEvDataShard::TEvSchemaChanged::TPtr& ev) { + return {ev->Get()->GetGeneration()}; + } + static bool HasOpResult(const TEvDataShard::TEvSchemaChanged::TPtr& ev) { + return ev->Get()->Record.HasOpResult(); + } + static TString GetName() { + return "TEvDataShard::TEvSchemaChanged"; + } +}; + +template<> +struct TEvSchemaChangedTraits { + static TActorId GetSource(const TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev) { + return TActorId{ev->Sender}; + } + static std::optional GetGeneration(const TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& /* ev */) { + return std::nullopt; //TODO consider to add generation to TEvColumnShard::TEvNotifyTxCompletionResult + } + static bool HasOpResult(const TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& /* ev */) { + return false; + } + static TString GetName() { + return "TEvColumnShard::TEvNotifyTxCompletionResult"; + } +}; + TSet AllIncomingEvents(); void IncParentDirAlterVersionWithRepublishSafeWithUndo(const TOperationId& opId, const TPath& path, TSchemeShard* ss, TSideEffects& onComplete); @@ -37,6 +72,7 @@ namespace NTableState { bool CollectProposeTransactionResults(const TOperationId& operationId, const TEvDataShard::TEvProposeTransactionResult__HandlePtr& ev, TOperationContext& context); bool CollectProposeTransactionResults(const TOperationId& operationId, const TEvColumnShard::TEvProposeTransactionResult__HandlePtr& ev, TOperationContext& context); bool CollectSchemaChanged(const TOperationId& operationId, const TEvDataShard::TEvSchemaChanged__HandlePtr& ev, TOperationContext& context); +bool CollectSchemaChanged(const TOperationId& operationId, const TEvColumnShard::TEvNotifyTxCompletionResult__HandlePtr& ev, TOperationContext& context); void SendSchemaChangedNotificationAck(const TOperationId& operationId, TActorId ackTo, TShardIdx shardIdx, TOperationContext& context); void AckAllSchemaChanges(const TOperationId& operationId, TTxState& txState, TOperationContext& context); @@ -59,12 +95,15 @@ class TProposedWaitParts: public TSubOperationState { << "NTableState::TProposedWaitParts" << " operationId# " << OperationId; } + template + bool HandleReplyImpl(const TEvent& ev, TOperationContext& context); public: TProposedWaitParts(TOperationId id, TTxState::ETxState nextState = TTxState::Done); bool ProgressState(TOperationContext& context) override; bool HandleReply(TEvDataShard::TEvSchemaChanged__HandlePtr& ev, TOperationContext& context) override; + bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult__HandlePtr& ev, TOperationContext& context) override; }; } // namespace NTableState diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp index b74177f65399..6f9e266a5f51 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_table.cpp @@ -26,7 +26,8 @@ class TConfigureParts: public TSubOperationState { IgnoreMessages(DebugHint(), {}); } - bool HandleReply(TEvDataShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { + template + bool HandleReplyImpl(TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, @@ -48,6 +49,14 @@ class TConfigureParts: public TSubOperationState { return true; } + bool HandleReply(TEvDataShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + + bool HandleReply(TEvColumnShard::TEvProposeTransactionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + bool ProgressState(TOperationContext& context) override { TTabletId ssId = context.SS->SelfTabletId(); @@ -64,18 +73,33 @@ class TConfigureParts: public TSubOperationState { Y_ABORT_UNLESS(dstPath.IsResolved()); TPath srcPath = TPath::Init(txState->SourcePathId, context.SS); Y_ABORT_UNLESS(srcPath.IsResolved()); - TTableInfo::TPtr srcTable = context.SS->Tables.at(srcPath->PathId); + Y_ABORT_UNLESS(srcPath->IsTable() || srcPath->IsColumnTable()); NIceDb::TNiceDb db(context.GetDB()); // txState catches table shards if (!txState->Shards) { - txState->Shards.reserve(srcTable->GetPartitions().size()); - for (const auto& shard : srcTable->GetPartitions()) { - auto shardIdx = shard.ShardIdx; + std::vector shardIdxs; + if (srcPath->IsTable()) { + const auto& srcTable = context.SS->Tables.at(srcPath->PathId); + shardIdxs.reserve(srcTable->GetPartitions().size()); + for (const auto& shard : srcTable->GetPartitions()) { + shardIdxs.emplace_back(shard.ShardIdx); + } + } else if (srcPath->IsColumnTable()) { + const auto& srcTable =context.SS->ColumnTables.GetVerified(srcPath.Base()->PathId); + shardIdxs.reserve(srcTable->GetShardIdsSet().size()); + for (const auto& id: srcTable->GetShardIdsSet()) { + shardIdxs.emplace_back(context.SS->TabletIdToShardIdx.at(TTabletId(id))); + } + } else { + Y_ABORT(); + } + const auto tabletType = srcPath->IsTable() ? ETabletType::DataShard : ETabletType::ColumnShard; + for (const auto& shardIdx : shardIdxs) { TShardInfo& shardInfo = context.SS->ShardInfos[shardIdx]; - txState->Shards.emplace_back(shardIdx, ETabletType::DataShard, TTxState::ConfigureParts); + txState->Shards.emplace_back(shardIdx, tabletType, TTxState::ConfigureParts); shardInfo.CurrentTxId = OperationId.GetTxId(); context.SS->PersistShardTx(db, shardIdx, OperationId.GetTxId()); @@ -84,9 +108,11 @@ class TConfigureParts: public TSubOperationState { } Y_ABORT_UNLESS(txState->Shards.size()); + const auto& seqNo = context.SS->StartRound(*txState); + TString txBody; - { - auto seqNo = context.SS->StartRound(*txState); + if (srcPath->IsTable()) { + TTableInfo::TPtr srcTable = context.SS->Tables.at(srcPath->PathId); NKikimrTxDataShard::TFlatSchemeTransaction tx; context.SS->FillSeqNo(tx, seqNo); @@ -117,20 +143,25 @@ class TConfigureParts: public TSubOperationState { srcChildPath->PathId.ToProto(remap->MutableSrcPathId()); dstIndexPath->PathId.ToProto(remap->MutableDstPathId()); } - Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); + } else if (srcPath->IsColumnTable()) { + NKikimrTxColumnShard::TSchemaTxBody tx; + context.SS->FillSeqNo(tx, seqNo); + auto move = tx.MutableMoveTable(); + move->SetSrcPathId(srcPath->PathId.LocalPathId); + move->SetDstPathId(dstPath->PathId.LocalPathId); + Y_PROTOBUF_SUPPRESS_NODISCARD tx.SerializeToString(&txBody); + } else { + Y_ABORT(); } - // send messages txState->ClearShardsInProgress(); - for (ui32 i = 0; i < txState->Shards.size(); ++i) { - auto idx = txState->Shards[i].Idx; - auto datashardId = context.SS->ShardInfos[idx].TabletID; - - auto event = context.SS->MakeDataShardProposal(txState->TargetPathId, OperationId, txBody, context.Ctx); - context.OnComplete.BindMsgToPipe(OperationId, datashardId, idx, event.Release()); + for (const auto& shard: txState->Shards) { + auto idx = shard.Idx; + auto tabletId = context.SS->ShardInfos[idx].TabletID; + auto event = context.SS->MakeShardProposal(dstPath, OperationId, seqNo, txBody, context.Ctx); + context.OnComplete.BindMsgToPipe(OperationId, tabletId, idx, event.Release()); } - txState->UpdateShardsInProgress(TTxState::ConfigureParts); return false; } @@ -148,7 +179,9 @@ void MarkSrcDropped(NIceDb::TNiceDb& db, srcPath->SetDropped(txState.PlanStep, operationId.GetTxId()); context.SS->PersistDropStep(db, srcPath->PathId, txState.PlanStep, operationId); - context.SS->Tables.at(srcPath->PathId)->DetachShardsStats(); + if (srcPath->IsTable()) { + context.SS->Tables.at(srcPath->PathId)->DetachShardsStats(); + } context.SS->PersistRemoveTable(db, srcPath->PathId, context.Ctx); context.SS->PersistUserAttributes(db, srcPath->PathId, srcPath->UserAttrs, nullptr); @@ -170,18 +203,19 @@ class TPropose: public TSubOperationState { : OperationId(id) , NextState(nextState) { - IgnoreMessages(DebugHint(), {TEvHive::TEvCreateTabletReply::EventType, TEvDataShard::TEvProposeTransactionResult::EventType}); + IgnoreMessages(DebugHint(), {TEvHive::TEvCreateTabletReply::EventType, TEvDataShard::TEvProposeTransactionResult::EventType, TEvColumnShard::TEvProposeTransactionResult::EventType}); } - bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + template + bool HandleReplyImpl(TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); const auto& evRecord = ev->Get()->Record; LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " at tablet: " << ssId); LOG_DEBUG_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " triggered early" << ", message: " << evRecord.ShortDebugString()); @@ -189,6 +223,14 @@ class TPropose: public TSubOperationState { return false; } + bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + + bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + bool HandleReply(TEvPrivate::TEvOperationPlan::TPtr& ev, TOperationContext& context) override { TStepId step = TStepId(ev->Get()->StepId); TTabletId ssId = context.SS->SelfTabletId(); @@ -225,16 +267,25 @@ class TPropose: public TSubOperationState { } Y_ABORT_UNLESS(!context.SS->Tables.contains(dstPath.Base()->PathId)); - Y_ABORT_UNLESS(context.SS->Tables.contains(srcPath.Base()->PathId)); - - TTableInfo::TPtr tableInfo = TTableInfo::DeepCopy(*context.SS->Tables.at(srcPath.Base()->PathId)); - tableInfo->ResetDescriptionCache(); - tableInfo->AlterVersion += 1; - - // copy table info - context.SS->Tables[dstPath.Base()->PathId] = tableInfo; - context.SS->PersistTable(db, dstPath.Base()->PathId); - context.SS->PersistTablePartitionStats(db, dstPath.Base()->PathId, tableInfo); + if (srcPath->IsTable()) { + Y_ABORT_UNLESS(context.SS->Tables.contains(srcPath.Base()->PathId)); + + TTableInfo::TPtr tableInfo = TTableInfo::DeepCopy(*context.SS->Tables.at(srcPath.Base()->PathId)); + tableInfo->ResetDescriptionCache(); + tableInfo->AlterVersion += 1; + + // copy table info + context.SS->Tables[dstPath.Base()->PathId] = tableInfo; + context.SS->PersistTable(db, dstPath.Base()->PathId); + context.SS->PersistTablePartitionStats(db, dstPath.Base()->PathId, tableInfo); + } else if (srcPath->IsColumnTable()) { + auto srcTable = context.SS->ColumnTables.GetVerified(srcPath.Base()->PathId); + auto tableInfo = context.SS->ColumnTables.BuildNew(dstPath.Base()->PathId, srcTable.GetPtr()); + tableInfo->AlterVersion += 1; + context.SS->PersistColumnTable(db, dstPath.Base()->PathId, *tableInfo, false); + } else { + Y_ABORT(); + } context.SS->IncrementPathDbRefCount(dstPath.Base()->PathId, "move table info"); dstPath->StepCreated = step; @@ -266,6 +317,11 @@ class TPropose: public TSubOperationState { for (const auto& shard : txState->Shards) { TShardIdx idx = shard.Idx; TTabletId tablet = context.SS->ShardInfos.at(idx).TabletID; + TPath srcPath = TPath::Init(txState->SourcePathId, context.SS); + if (srcPath->IsColumnTable()) { + auto event = std::make_unique(ui64(OperationId.GetTxId())); + context.OnComplete.BindMsgToPipe(OperationId, tablet, shard.Idx, event.release()); + } shardSet.insert(tablet); } @@ -293,17 +349,25 @@ class TWaitRenamedPathPublication: public TSubOperationState { IgnoreMessages(DebugHint(), {TEvHive::TEvCreateTabletReply::EventType, TEvDataShard::TEvProposeTransactionResult::EventType, TEvPrivate::TEvOperationPlan::EventType}); } - bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + template + bool HandleReplyImpl(TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvDataShard::TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << ", save it" << ", at schemeshard: " << ssId); NTableState::CollectSchemaChanged(OperationId, ev, context); return false; } + bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + + bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } bool HandleReply(TEvPrivate::TEvCompletePublication::TPtr& ev, TOperationContext& context) override { TTabletId ssId = context.SS->SelfTabletId(); @@ -373,11 +437,12 @@ class TDeleteTableBarrier: public TSubOperationState { IgnoreMessages(DebugHint(), {TEvHive::TEvCreateTabletReply::EventType, TEvDataShard::TEvProposeTransactionResult::EventType, TEvPrivate::TEvOperationPlan::EventType}); } - bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + template + bool HandleReplyImpl(TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TEvDataShard::TEvSchemaChanged" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << ", save it" << ", at schemeshard: " << ssId); @@ -385,6 +450,14 @@ class TDeleteTableBarrier: public TSubOperationState { return false; } + bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + + bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + bool HandleReply(TEvPrivate::TEvCompleteBarrier::TPtr& ev, TOperationContext& context) override { TTabletId ssId = context.SS->SelfTabletId(); @@ -404,20 +477,21 @@ class TDeleteTableBarrier: public TSubOperationState { Y_ABORT_UNLESS(txState->PlanStep); MarkSrcDropped(db, context, OperationId, *txState, srcPath); - - Y_ABORT_UNLESS(context.SS->Tables.contains(dstPath.Base()->PathId)); - auto tableInfo = context.SS->Tables.at(dstPath.Base()->PathId); - - if (tableInfo->IsTTLEnabled() && !context.SS->TTLEnabledTables.contains(dstPath.Base()->PathId)) { - context.SS->TTLEnabledTables[dstPath.Base()->PathId] = tableInfo; - // MarkSrcDropped() removes srcPath from TTLEnabledTables & decrements the counters - context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); - - const auto now = context.Ctx.Now(); - for (auto& shard : tableInfo->GetPartitions()) { - auto& lag = shard.LastCondEraseLag; - lag = now - shard.LastCondErase; - context.SS->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); + if (srcPath->IsTable()) { + Y_ABORT_UNLESS(context.SS->Tables.contains(dstPath.Base()->PathId)); + auto tableInfo = context.SS->Tables.at(dstPath.Base()->PathId); + + if (tableInfo->IsTTLEnabled() && !context.SS->TTLEnabledTables.contains(dstPath.Base()->PathId)) { + context.SS->TTLEnabledTables[dstPath.Base()->PathId] = tableInfo; + // MarkSrcDropped() removes srcPath from TTLEnabledTables & decrements the counters + context.SS->TabletCounters->Simple()[COUNTER_TTL_ENABLED_TABLE_COUNT].Add(1); + + const auto now = context.Ctx.Now(); + for (auto& shard : tableInfo->GetPartitions()) { + auto& lag = shard.LastCondEraseLag; + lag = now - shard.LastCondErase; + context.SS->TabletCounters->Percentile()[COUNTER_NUM_SHARDS_BY_TTL_LAG].IncrementFor(lag->Seconds()); + } } } @@ -457,13 +531,13 @@ class TDone: public TSubOperationState { { IgnoreMessages(DebugHint(), AllIncomingEvents()); } - - bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + template + bool HandleReplyImpl(TEvent& ev, TOperationContext& context) { TTabletId ssId = context.SS->SelfTabletId(); const TActorId& ackTo = ev->Sender; LOG_INFO_S(context.Ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, - DebugHint() << " HandleReply TProposedDeletePart" + DebugHint() << " HandleReply " << TEvSchemaChangedTraits::GetName() << " repeated message, ack it anyway" << " at tablet: " << ssId); @@ -473,6 +547,14 @@ class TDone: public TSubOperationState { context.OnComplete.Send(ackTo, std::move(event)); return false; } + + bool HandleReply(TEvDataShard::TEvSchemaChanged::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } + + bool HandleReply(TEvColumnShard::TEvNotifyTxCompletionResult::TPtr& ev, TOperationContext& context) override { + return HandleReplyImpl(ev, context); + } bool ProgressState(TOperationContext& context) override { TTabletId ssId = context.SS->SelfTabletId(); @@ -576,6 +658,9 @@ class TMoveTable: public TSubOperation { TPath srcPath = TPath::Resolve(srcPathStr, context.SS); { + if (!srcPath->IsTable() && !srcPath->IsColumnTable()) { + result->SetError(NKikimrScheme::StatusPreconditionFailed, "Cannot move non-tables"); + } TPath::TChecker checks = srcPath.Check(); checks .NotEmpty() @@ -583,7 +668,6 @@ class TMoveTable: public TSubOperation { .IsAtLocalSchemeShard() .IsResolved() .NotDeleted() - .IsTable() .NotBackupTable() .NotAsyncReplicaTable() .NotUnderTheSameOperation(OperationId.GetTxId()) @@ -653,7 +737,7 @@ class TMoveTable: public TSubOperation { } else { checks .NotUnderTheSameOperation(OperationId.GetTxId()) - .FailOnExist(TPathElement::EPathType::EPathTypeTable, acceptExisted); + .FailOnExist(srcPath->IsColumnTable() ? TPathElement::EPathType::EPathTypeColumnTable : TPathElement::EPathType::EPathTypeTable, acceptExisted); } } else { checks @@ -705,7 +789,7 @@ class TMoveTable: public TSubOperation { dstPath.Base()->CreateTxId = OperationId.GetTxId(); dstPath.Base()->LastTxId = OperationId.GetTxId(); dstPath.Base()->PathState = TPathElement::EPathState::EPathStateCreate; - dstPath.Base()->PathType = TPathElement::EPathType::EPathTypeTable; + dstPath.Base()->PathType = srcPath.Base()->PathType; dstPath.Base()->UserAttrs->AlterData = srcPath.Base()->UserAttrs; dstPath.Base()->ACL = srcPath.Base()->ACL; @@ -722,11 +806,12 @@ class TMoveTable: public TSubOperation { IncParentDirAlterVersionWithRepublishSafeWithUndo(OperationId, srcPath, context.SS, context.OnComplete); // wait splits - TTableInfo::TPtr tableSrc = context.SS->Tables.at(srcPath.Base()->PathId); - for (auto splitTx: tableSrc->GetSplitOpsInFlight()) { - context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); + if (srcPath->IsTable()) { + TTableInfo::TPtr tableSrc = context.SS->Tables.at(srcPath.Base()->PathId); + for (auto splitTx: tableSrc->GetSplitOpsInFlight()) { + context.OnComplete.Dependence(splitTx.GetTxId(), OperationId.GetTxId()); + } } - context.OnComplete.ActivateTx(OperationId); SetState(NextState()); diff --git a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp index c3c8cde8247d..f4b26cbb72d1 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation_move_tables.cpp @@ -31,10 +31,12 @@ TVector CreateConsistentMoveTable(TOperationId nextId, cons TPath srcPath = TPath::Resolve(srcStr, context.SS); { + if (!srcPath->IsTable() && !srcPath->IsColumnTable()) { + return {CreateReject(nextId, NKikimrScheme::StatusPreconditionFailed, "Cannot move non-tables")}; + } TPath::TChecker checks = srcPath.Check(); checks.IsResolved() .NotDeleted() - .IsTable() .NotAsyncReplicaTable() .IsCommonSensePath(); diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 29ddc5165afa..1fb2d4d43ce6 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -289,6 +289,30 @@ THolder TSchemeShard::MakeDataShardProposal ); } +THolder TSchemeShard::MakeColumnShardProposal( + const TPathId& pathId, const TOperationId& opId, + const TMessageSeqNo& seqNo, const TString& body, const TActorContext& ctx) const +{ + return MakeHolder( + NKikimrTxColumnShard::TX_KIND_SCHEMA, TabletID(), ctx.SelfID, + ui64(opId.GetTxId()), body, seqNo, SelectProcessingParams(pathId), + 0, 0 + ); +} + +THolder<::NActors::IEventBase> TSchemeShard::MakeShardProposal( + const TPath& path, const TOperationId& opId, + const TMessageSeqNo& seqNo, const TString& body, const TActorContext& ctx) const +{ + if (path->IsTable()) { + return MakeDataShardProposal(path->PathId, opId, body, ctx); + } else if (path->IsColumnTable()) { + return MakeColumnShardProposal(path->PathId, opId, seqNo, body, ctx); + } else { + Y_ABORT(); + } +} + TTxId TSchemeShard::GetCachedTxId(const TActorContext &ctx) { TTxId txId = InvalidTxId; if (CachedTxIds) { diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.h b/ydb/core/tx/schemeshard/schemeshard_impl.h index d0025567238a..3cf0c1a39826 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.h +++ b/ydb/core/tx/schemeshard/schemeshard_impl.h @@ -404,6 +404,12 @@ class TSchemeShard THolder MakeDataShardProposal(const TPathId& pathId, const TOperationId& opId, const TString& body, const TActorContext& ctx) const; + THolder MakeColumnShardProposal(const TPathId& pathId, const TOperationId& opId, + const TMessageSeqNo& seqNo, const TString& body, const TActorContext& ctx) const; + + THolder<::NActors::IEventBase> MakeShardProposal(const TPath& path, const TOperationId& opId, + const TMessageSeqNo& seqNo, const TString& body, const TActorContext& ctx) const; + TPathId RootPathId() const { return MakeLocalId(TPathElement::RootPathId); From 820f81f3e6810f80e236b04e5bf772b685ff10ec Mon Sep 17 00:00:00 2001 From: xyliganSereja Date: Thu, 19 Jun 2025 15:56:36 +0300 Subject: [PATCH 26/62] add format for slice_limiter (#19870) --- .../columnshard/columnshard__write_index.cpp | 8 +++---- ydb/core/tx/limiter/service/service.cpp | 8 +++---- ydb/core/tx/limiter/service/service.h | 22 +++++++++---------- ydb/core/tx/limiter/usage/abstract.h | 10 ++++----- ydb/core/tx/limiter/usage/config.cpp | 3 ++- ydb/core/tx/limiter/usage/config.h | 5 ++++- ydb/core/tx/limiter/usage/events.h | 7 ++++-- ydb/core/tx/limiter/usage/service.h | 10 +++++---- 8 files changed, 39 insertions(+), 34 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard__write_index.cpp b/ydb/core/tx/columnshard/columnshard__write_index.cpp index 54672008be50..87cdc9909d34 100644 --- a/ydb/core/tx/columnshard/columnshard__write_index.cpp +++ b/ydb/core/tx/columnshard/columnshard__write_index.cpp @@ -1,8 +1,8 @@ #include "columnshard_impl.h" +#include "columnshard_private_events.h" #include "blobs_action/transaction/tx_draft.h" #include "blobs_action/transaction/tx_write_index.h" -#include "columnshard_private_events.h" #include "engines/changes/abstract/abstract.h" #include "engines/writer/compacted_blob_constructor.h" @@ -32,9 +32,7 @@ class TDiskResourcesRequest: public NLimiter::IResourceRequest { : TBase(writeController->GetWriteVolume()) , WriteController(writeController) , Changes(changes) - , TabletId(tabletId) - { - + , TabletId(tabletId) { } }; @@ -84,4 +82,4 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteIndex::TPtr& ev, const TActorConte } } -} +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/limiter/service/service.cpp b/ydb/core/tx/limiter/service/service.cpp index e31ea0f253c7..0105cfcbb3a7 100644 --- a/ydb/core/tx/limiter/service/service.cpp +++ b/ydb/core/tx/limiter/service/service.cpp @@ -5,8 +5,7 @@ namespace NKikimr::NLimiter { TLimiterActor::TLimiterActor(const TConfig& config, const TString& limiterName, TIntrusivePtr<::NMonitoring::TDynamicCounters> baseCounters) : LimiterName(limiterName) , Config(config) - , Counters(LimiterName, baseCounters) -{ + , Counters(LimiterName, baseCounters) { Counters.InProgressLimit->Set(Config.GetLimit()); } @@ -41,7 +40,8 @@ void TLimiterActor::HandleMain(NActors::TEvents::TEvWakeup::TPtr& /*ev*/) { if (RequestsInFlight.empty()) { AFL_VERIFY(!VolumeInFlight); } - while (RequestsQueue.size() && (RequestsInFlight.empty() || VolumeInFlight + RequestsQueue.front().GetRequest()->GetVolume() <= Config.GetLimit())) { + while (RequestsQueue.size() && + (RequestsInFlight.empty() || VolumeInFlight + RequestsQueue.front().GetRequest()->GetVolume() <= Config.GetLimit())) { Counters.WaitingHistogram->Collect((i64)(now - RequestsQueue.front().GetInstant()).MilliSeconds(), 1); VolumeInFlight += RequestsQueue.front().GetRequest()->GetVolume(); RequestsInFlight.emplace_back(now, RequestsQueue.front().GetRequest()->GetVolume()); @@ -60,4 +60,4 @@ void TLimiterActor::HandleMain(NActors::TEvents::TEvWakeup::TPtr& /*ev*/) { Counters.WaitingQueueVolume->Set(VolumeInWaiting); } -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/service/service.h b/ydb/core/tx/limiter/service/service.h index dd7fd10f3d49..4082379e2086 100644 --- a/ydb/core/tx/limiter/service/service.h +++ b/ydb/core/tx/limiter/service/service.h @@ -1,22 +1,21 @@ #pragma once -#include #include #include #include +#include #include #include -#include +#include #include -#include - namespace NKikimr::NLimiter { class TCounters: public NColumnShard::TCommonCountersOwner { private: using TBase = NColumnShard::TCommonCountersOwner; + public: const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueCount; const ::NMonitoring::TDynamicCounters::TCounterPtr WaitingQueueVolume; @@ -50,11 +49,11 @@ class TLimiterActor: public NActors::TActorBootstrapped { private: YDB_READONLY(TMonotonic, Instant, TMonotonic::Zero()); YDB_READONLY_DEF(std::shared_ptr, Request); + public: TResourceRequest(const TMonotonic instant, const std::shared_ptr& req) : Instant(instant) , Request(req) { - } }; @@ -62,11 +61,11 @@ class TLimiterActor: public NActors::TActorBootstrapped { private: YDB_READONLY(TMonotonic, Instant, TMonotonic::Zero()); YDB_READONLY(ui64, Volume, 0); + public: TResourceRequestInFlight(const TMonotonic instant, const ui64 volume) : Instant(instant) , Volume(volume) { - } }; @@ -79,15 +78,14 @@ class TLimiterActor: public NActors::TActorBootstrapped { void HandleMain(NActors::TEvents::TEvWakeup::TPtr& ev); public: - STATEFN(StateMain) { switch (ev->GetTypeRewrite()) { hFunc(TEvExternal::TEvAskResource, HandleMain); hFunc(NActors::TEvents::TEvWakeup, HandleMain); - default: - AFL_ERROR(NKikimrServices::TX_LIMITER)("limiter", LimiterName)("problem", "unexpected event")("type", ev->GetTypeRewrite()); - AFL_VERIFY_DEBUG(false)("type", ev->GetTypeRewrite()); - break; + default: + AFL_ERROR(NKikimrServices::TX_LIMITER)("limiter", LimiterName)("problem", "unexpected event")("type", ev->GetTypeRewrite()); + AFL_VERIFY_DEBUG(false)("type", ev->GetTypeRewrite()); + break; } } @@ -98,4 +96,4 @@ class TLimiterActor: public NActors::TActorBootstrapped { } }; -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/usage/abstract.h b/ydb/core/tx/limiter/usage/abstract.h index 3a4b45983b5b..4479e54f6ad2 100644 --- a/ydb/core/tx/limiter/usage/abstract.h +++ b/ydb/core/tx/limiter/usage/abstract.h @@ -1,12 +1,14 @@ #pragma once -#include #include +#include + namespace NKikimr::NLimiter { class IResourceRequest { private: YDB_READONLY(ui64, Volume, 0); virtual void DoOnResourceAllocated() = 0; + public: void OnResourceAllocated() { return DoOnResourceAllocated(); @@ -15,10 +17,8 @@ class IResourceRequest { virtual ~IResourceRequest() = default; IResourceRequest(const ui64 volume) - : Volume(volume) - { - + : Volume(volume) { } }; -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/usage/config.cpp b/ydb/core/tx/limiter/usage/config.cpp index a4448ea10320..ae8537a45fdd 100644 --- a/ydb/core/tx/limiter/usage/config.cpp +++ b/ydb/core/tx/limiter/usage/config.cpp @@ -1,4 +1,5 @@ #include "config.h" + #include namespace NKikimr::NLimiter { @@ -9,4 +10,4 @@ TString TConfig::DebugString() const { return sb; } -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/usage/config.h b/ydb/core/tx/limiter/usage/config.h index 434f66f2d485..67e9ce665412 100644 --- a/ydb/core/tx/limiter/usage/config.h +++ b/ydb/core/tx/limiter/usage/config.h @@ -1,6 +1,8 @@ #pragma once #include + #include + #include namespace NKikimr::NLimiter { @@ -10,6 +12,7 @@ class TConfig { YDB_READONLY(TDuration, Period, TDuration::Seconds(1)); YDB_READONLY(ui64, Limit, 0); YDB_READONLY_FLAG(Enabled, true); + public: template bool DeserializeFromProto(const NKikimrConfig::TLimiterConfig& config) { @@ -34,4 +37,4 @@ class TConfig { TString DebugString() const; }; -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/usage/events.h b/ydb/core/tx/limiter/usage/events.h index 8b5b3a248e65..7264e3725ac1 100644 --- a/ydb/core/tx/limiter/usage/events.h +++ b/ydb/core/tx/limiter/usage/events.h @@ -1,9 +1,11 @@ #pragma once #include "abstract.h" + +#include + #include #include #include -#include namespace NKikimr::NLimiter { @@ -18,6 +20,7 @@ struct TEvExternal { class TEvAskResource: public NActors::TEventLocal { private: YDB_READONLY_DEF(std::shared_ptr, Request); + public: TEvAskResource() = default; @@ -27,4 +30,4 @@ struct TEvExternal { }; }; -} +} // namespace NKikimr::NLimiter diff --git a/ydb/core/tx/limiter/usage/service.h b/ydb/core/tx/limiter/usage/service.h index b0b435fa802f..7ac3946e3995 100644 --- a/ydb/core/tx/limiter/usage/service.h +++ b/ydb/core/tx/limiter/usage/service.h @@ -1,10 +1,12 @@ #pragma once #include "config.h" -#include -#include + #include #include +#include +#include + namespace NKikimr::NLimiter { template @@ -19,6 +21,7 @@ class TServiceOperatorImpl { Y_ABORT_UNLESS(TLimiterPolicy::Name.size() == 4); return TLimiterPolicy::Name; } + public: static bool AskResource(const std::shared_ptr& request) { AFL_VERIFY(!!request); @@ -42,7 +45,6 @@ class TServiceOperatorImpl { Register(config); return new TLimiterActor(config, GetLimiterName(), baseSignals); } - }; class TCompDiskLimiterPolicy { @@ -55,4 +57,4 @@ class TCompDiskLimiterPolicy { using TCompDiskOperator = TServiceOperatorImpl; -} +} // namespace NKikimr::NLimiter From 287e48f8a6d85d956ef1080fe94a45cf60010225 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 19 Jun 2025 18:59:24 +0300 Subject: [PATCH 27/62] correct cpu conveyor (#19838) --- ydb/core/tx/columnshard/columnshard_impl.cpp | 22 +++++++ .../tx/columnshard/data_reader/contexts.h | 17 ++++- .../tx/columnshard/data_reader/fetcher.cpp | 12 +--- ydb/core/tx/columnshard/data_reader/fetcher.h | 5 ++ .../columnshard/data_reader/fetching_steps.h | 9 +-- .../conveyor_composite/service/category.cpp | 66 +++++++++++++------ .../tx/conveyor_composite/service/category.h | 16 +++-- .../tx/conveyor_composite/service/common.cpp | 12 +++- .../tx/conveyor_composite/service/common.h | 26 ++++++-- .../conveyor_composite/service/counters.cpp | 4 +- .../tx/conveyor_composite/service/counters.h | 2 +- .../tx/conveyor_composite/service/manager.h | 6 -- .../tx/conveyor_composite/service/process.h | 23 +++++-- .../tx/conveyor_composite/service/service.cpp | 7 -- .../tx/conveyor_composite/service/service.h | 2 - .../service/workers_pool.cpp | 6 +- 16 files changed, 156 insertions(+), 79 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 440c728c0310..2342fb69fb8d 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -572,6 +572,28 @@ class TCompactionExecutor: public NOlap::NDataFetcher::IFetchCallback { } TActorContext::AsActorContext().Send(ParentActorId, std::move(ev)); } + + virtual ui64 GetNecessaryDataMemory(const std::shared_ptr& columnIds, + const std::vector& acc) const override { + AFL_VERIFY(!columnIds); + THashMap memoryByColumns; + for (auto&& a : acc) { + THashMap memoryByPortionColumns; + for (auto&& c : a.GetRecordsVerified()) { + const ui64 current = memoryByPortionColumns[c.GetEntityId()]; + memoryByPortionColumns[c.GetEntityId()] = std::max(current, c.GetMeta().GetRawBytes()); + } + for (auto&& c : memoryByPortionColumns) { + memoryByColumns[c.first] += c.second; + } + } + ui64 max = 0; + for (auto&& c : memoryByColumns) { + max = std::max(max, c.second); + } + return max; + } + virtual void DoOnError(const TString& errorMessage) override { auto ev = std::make_unique(Changes, false); ev->SetPutStatus(NKikimrProto::ERROR); diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h index 203dabf6ef34..259998dba5cc 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.h +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -68,7 +69,7 @@ class TCurrentContext: TMoveOnly { TCurrentContext() { static std::shared_ptr stageFeatures = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - + MemoryProcessGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(MemoryProcessId, { stageFeatures }); MemoryProcessScopeGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildScopeGuard(MemoryProcessId, 1); MemoryProcessGroupGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildGroupGuard(MemoryProcessId, 1); @@ -105,6 +106,19 @@ class IFetchCallback { public: virtual ~IFetchCallback() = default; + virtual ui64 GetNecessaryDataMemory( + const std::shared_ptr& columnIds, const std::vector& acc) const { + ui64 memory = 0; + for (auto&& a : acc) { + if (columnIds) { + memory += a.GetColumnBlobBytes(columnIds->GetColumnIds()); + } else { + memory += a.GetPortionInfo().GetTotalBlobBytes(); + } + } + return memory; + } + virtual std::optional GetMemoryForUsage() const { return std::nullopt; } @@ -113,7 +127,6 @@ class IFetchCallback { virtual TString GetClassName() const = 0; virtual void OnStageStarting(const EFetchingStage /*stage*/) { - } void OnFinished(TCurrentContext&& context) { diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.cpp b/ydb/core/tx/columnshard/data_reader/fetcher.cpp index c94cb5e6820e..b60c0d7f0ebb 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.cpp +++ b/ydb/core/tx/columnshard/data_reader/fetcher.cpp @@ -10,11 +10,7 @@ void TPortionsDataFetcher::StartColumnsFetching(TRequestInput&& input, std::shar std::vector> steps; steps.emplace_back(std::make_shared()); steps.emplace_back(std::make_shared()); - if (auto mem = callback->GetMemoryForUsage()) { - steps.emplace_back(std::make_shared(entityIds, *mem)); - } else { - steps.emplace_back(std::make_shared(entityIds)); - } + steps.emplace_back(std::make_shared(entityIds)); steps.emplace_back(std::make_shared(entityIds)); return std::make_shared(std::move(steps), "PARTIAL_PORTIONS_FETCHING::" + ::ToString(input.GetConsumer())); }(); @@ -28,11 +24,7 @@ void TPortionsDataFetcher::StartFullPortionsFetching(TRequestInput&& input, std: std::vector> steps; steps.emplace_back(std::make_shared()); steps.emplace_back(std::make_shared()); - if (auto mem = callback->GetMemoryForUsage()) { - steps.emplace_back(std::make_shared(nullptr, *mem)); - } else { - steps.emplace_back(std::make_shared(nullptr)); - } + steps.emplace_back(std::make_shared(nullptr)); steps.emplace_back(std::make_shared(nullptr)); return std::make_shared(std::move(steps), "FULL_PORTIONS_FETCHING::" + ::ToString(input.GetConsumer())); }(); diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index b94cfd5f71ad..f44d59a307e5 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -68,6 +68,11 @@ class TPortionsDataFetcher: TNonCopyable { CurrentContext.GetMemoryProcessId(), CurrentContext.GetMemoryScopeId(), CurrentContext.GetMemoryGroupId(), { task }, 0); } + ui64 GetNecessaryDataMemory( + const std::shared_ptr& columnIds, const std::vector& acc) const { + return Callback->GetNecessaryDataMemory(columnIds, acc); + } + TPortionsDataFetcher(TRequestInput&& input, std::shared_ptr&& callback, const std::shared_ptr& environment, const std::shared_ptr& script, const NConveyorComposite::ESpecialTaskCategory conveyorCategory) : Input(std::move(input)) diff --git a/ydb/core/tx/columnshard/data_reader/fetching_steps.h b/ydb/core/tx/columnshard/data_reader/fetching_steps.h index f78fc962371d..1f01404a03a1 100644 --- a/ydb/core/tx/columnshard/data_reader/fetching_steps.h +++ b/ydb/core/tx/columnshard/data_reader/fetching_steps.h @@ -131,14 +131,7 @@ class TAskDataResourceStep: public IFetchingStep { virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { fetchingContext->SetStage(EFetchingStage::AskDataResources); const std::vector& accessors = fetchingContext->GetCurrentContext().GetPortionAccessors(); - ui64 memory = 0; - for (auto&& i : accessors) { - if (ColumnIds) { - memory += i.GetColumnBlobBytes(ColumnIds->GetColumnIds()); - } else { - memory += i.GetPortionInfo().GetTotalBlobBytes(); - } - } + const ui64 memory = fetchingContext->GetNecessaryDataMemory(ColumnIds, accessors); if (!memory) { return IFetchingStep::EStepResult::Continue; } diff --git a/ydb/core/tx/conveyor_composite/service/category.cpp b/ydb/core/tx/conveyor_composite/service/category.cpp index e56a28103930..8d73af056d98 100644 --- a/ydb/core/tx/conveyor_composite/service/category.cpp +++ b/ydb/core/tx/conveyor_composite/service/category.cpp @@ -3,37 +3,35 @@ namespace NKikimr::NConveyorComposite { bool TProcessCategory::HasTasks() const { - return ProcessesWithTasks.size(); -} - -void TProcessCategory::DoQuant(const TMonotonic newStart) { - CPUUsage->Cut(newStart); - for (auto&& i : Processes) { - i.second->DoQuant(newStart); - } + return WeightedProcesses.size(); } std::optional TProcessCategory::ExtractTaskWithPrediction(const std::shared_ptr& counters, THashSet& scopeIds) { std::shared_ptr pMin; - TDuration dMin; - for (auto&& [_, p] : ProcessesWithTasks) { - if (!p->GetScope()->CheckToRun()) { - continue; + for (auto it = WeightedProcesses.begin(); it != WeightedProcesses.end(); ++it) { + for (ui32 i = 0; i < it->second.size(); ++i) { + if (!it->second[i]->GetScope()->CheckToRun()) { + continue; + } + pMin = it->second[i]; + std::swap(it->second[i], it->second.back()); + it->second.pop_back(); + if (it->second.empty()) { + WeightedProcesses.erase(it); + } + break; } - const TDuration d = p->GetCPUUsage()->CalcWeight(p->GetWeight()); - if (!pMin || d < dMin) { - dMin = d; - pMin = p; + if (pMin) { + break; } } if (!pMin) { return std::nullopt; } - AFL_VERIFY(pMin); auto result = pMin->ExtractTaskWithPrediction(counters); - if (pMin->GetTasksCount() == 0) { - AFL_VERIFY(ProcessesWithTasks.erase(pMin->GetProcessId())); - } + if (pMin->GetTasksCount()) { + WeightedProcesses[pMin->GetWeightedUsage()].emplace_back(pMin); + } if (scopeIds.emplace(pMin->GetScope()->GetScopeId()).second) { pMin->GetScope()->IncInFlight(); } @@ -98,7 +96,35 @@ void TProcessCategory::PutTaskResult(TWorkerTaskResult&& result, THashSetsecond)); it->second->PutTaskResult(std::move(result)); + if (it->second->GetTasksCount()) { + WeightedProcesses[it->second->GetWeightedUsage()].emplace_back(it->second); + } +} + +bool TProcessCategory::RemoveWeightedProcess(const std::shared_ptr& process) { + if (!process->GetTasksCount()) { + return false; + } + AFL_VERIFY(WeightedProcesses.size()); + auto itW = WeightedProcesses.find(process->GetWeightedUsage()); + AFL_VERIFY(itW != WeightedProcesses.end())("weight", process->GetWeightedUsage().GetValue())("size", WeightedProcesses.size())( + "first", WeightedProcesses.begin()->first.GetValue()); + for (ui32 i = 0; i < itW->second.size(); ++i) { + if (itW->second[i]->GetProcessId() != process->GetProcessId()) { + continue; + } + itW->second[i] = itW->second.back(); + if (itW->second.size() == 1) { + WeightedProcesses.erase(itW); + } else { + itW->second.pop_back(); + } + return true; + } + AFL_VERIFY(false); + return false; } } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/category.h b/ydb/core/tx/conveyor_composite/service/category.h index 94261a68170f..fec6763094c3 100644 --- a/ydb/core/tx/conveyor_composite/service/category.h +++ b/ydb/core/tx/conveyor_composite/service/category.h @@ -17,9 +17,11 @@ class TProcessCategory: public TNonCopyable { YDB_READONLY_DEF(std::shared_ptr, Counters); THashMap> Scopes; THashMap> Processes; - THashMap> ProcessesWithTasks; + std::map>> WeightedProcesses; const NConfig::TCategory Config; + [[nodiscard]] bool RemoveWeightedProcess(const std::shared_ptr& process); + public: ui32 GetWaitingQueueSize() const { return WaitingTasksCount->Val(); @@ -39,10 +41,13 @@ class TProcessCategory: public TNonCopyable { void RegisterTask(const ui64 internalProcessId, std::shared_ptr&& task) { auto it = Processes.find(internalProcessId); AFL_VERIFY(it != Processes.end())("process_id", internalProcessId); - it->second->RegisterTask(std::move(task), Category); - if (it->second->GetTasksCount() == 1) { - AFL_VERIFY(ProcessesWithTasks.emplace(internalProcessId, it->second).second); + if (!it->second->GetTasks().size()) { + if (!WeightedProcesses.empty() && !it->second->GetInProgressTasksCount()) { + it->second->SetBaseWeight(WeightedProcesses.begin()->first); + } + WeightedProcesses[it->second->GetWeightedUsage()].emplace_back(it->second); } + it->second->RegisterTask(std::move(task), Category); } void PutTaskResult(TWorkerTaskResult&& result, THashSet& scopeIds); @@ -55,7 +60,7 @@ class TProcessCategory: public TNonCopyable { void UnregisterProcess(const ui64 processId) { auto it = Processes.find(processId); AFL_VERIFY(it != Processes.end()); - ProcessesWithTasks.erase(processId); + Y_UNUSED(RemoveWeightedProcess(it->second)); if (it->second->GetScope()->DecProcesses()) { AFL_VERIFY(Scopes.erase(it->second->GetScope()->GetScopeId())); } @@ -67,7 +72,6 @@ class TProcessCategory: public TNonCopyable { } bool HasTasks() const; - void DoQuant(const TMonotonic newStart); std::optional ExtractTaskWithPrediction(const std::shared_ptr& counters, THashSet& scopeIds); TProcessScope& MutableProcessScope(const TString& scopeName); TProcessScope* MutableProcessScopeOptional(const TString& scopeName); diff --git a/ydb/core/tx/conveyor_composite/service/common.cpp b/ydb/core/tx/conveyor_composite/service/common.cpp index 766662a80bef..bc1e111dfca5 100644 --- a/ydb/core/tx/conveyor_composite/service/common.cpp +++ b/ydb/core/tx/conveyor_composite/service/common.cpp @@ -5,9 +5,14 @@ namespace NKikimr::NConveyorComposite { void TCPUUsage::Exchange(const TDuration predicted, const TMonotonic start, const TMonotonic finish) { - Usage.emplace_back(TTaskCPUUsage(start, finish)); + { + TTaskCPUUsage usage(start, finish); + if (usage.Cut(StartInstant)) { + Usage.emplace_back(TTaskCPUUsage(start, finish)); + Duration += Usage.back().GetDuration(); + } + } AFL_VERIFY(predicted <= PredictedDuration)("predicted_delta", predicted)("predicted_sum", PredictedDuration); - Duration += Usage.back().GetDuration(); PredictedDuration -= predicted; if (Parent) { Parent->Exchange(predicted, start, finish); @@ -15,6 +20,7 @@ void TCPUUsage::Exchange(const TDuration predicted, const TMonotonic start, cons } void TCPUUsage::Cut(const TMonotonic start) { + StartInstant = start; ui32 idx = 0; while (idx < Usage.size()) { AFL_VERIFY(Usage[idx].GetDuration() <= Duration); @@ -23,7 +29,7 @@ void TCPUUsage::Cut(const TMonotonic start) { std::swap(Usage[idx], Usage.back()); Usage.pop_back(); } else { - Usage[idx].Cut(start); + AFL_VERIFY(Usage[idx].Cut(start)); Duration += Usage[idx].GetDuration(); ++idx; } diff --git a/ydb/core/tx/conveyor_composite/service/common.h b/ydb/core/tx/conveyor_composite/service/common.h index 8faa4e1cc6b2..df42187b86ad 100644 --- a/ydb/core/tx/conveyor_composite/service/common.h +++ b/ydb/core/tx/conveyor_composite/service/common.h @@ -15,11 +15,14 @@ class TTaskCPUUsage { YDB_READONLY_DEF(TDuration, Duration); public: - void Cut(const TMonotonic start) { - AFL_VERIFY(start < Finish); + [[nodiscard]] bool Cut(const TMonotonic start) { + if (Finish <= start) { + return false; + } if (Start <= start) { Start = start; } + return true; } TTaskCPUUsage(const TMonotonic start, const TMonotonic finish) @@ -61,12 +64,20 @@ class TCPUUsage { YDB_READONLY_DEF(TDuration, Duration); YDB_READONLY_DEF(TDuration, PredictedDuration); std::shared_ptr Parent; + TMonotonic StartInstant = TMonotonic::Zero(); public: TCPUUsage(const std::shared_ptr& parent) : Parent(parent) { } + void Clear() { + Usage.clear(); + Duration = TDuration::Zero(); + PredictedDuration = TDuration::Zero(); + StartInstant = TMonotonic::Zero(); + } + TDuration CalcWeight(const double w) const { if (w <= 0) { return TDuration::Max(); @@ -82,11 +93,14 @@ class TCPUUsage { } } - void AddUsage(const TTaskCPUUsage& usage) { - Usage.emplace_back(usage); - Duration += usage.GetDuration(); + void AddUsage(const TTaskCPUUsage& extUsage) { + TTaskCPUUsage usage = extUsage; + if (usage.Cut(StartInstant)) { + Usage.emplace_back(usage); + Duration += usage.GetDuration(); + } if (Parent) { - Parent->AddUsage(usage); + Parent->AddUsage(extUsage); } } diff --git a/ydb/core/tx/conveyor_composite/service/counters.cpp b/ydb/core/tx/conveyor_composite/service/counters.cpp index 7db41fcc13cb..a63d8bd60da8 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.cpp +++ b/ydb/core/tx/conveyor_composite/service/counters.cpp @@ -18,14 +18,14 @@ TWorkersPoolCounters::TWorkersPoolCounters(const TString& poolName, const NColum , OverlimitRate(TBase::GetDeriviative("Overlimit")) , WaitWorkerRate(TBase::GetDeriviative("WaitWorker")) , UseWorkerRate(TBase::GetDeriviative("UseWorker")) - , ChangeCPULimitRate(TBase::GetDeriviative("ChangeCPULimit")) - , NoTasks(TBase::GetDeriviative("NoTasks")) { + , ChangeCPULimitRate(TBase::GetDeriviative("ChangeCPULimit")) { } TWPCategorySignals::TWPCategorySignals(NColumnShard::TCommonCountersOwner& base, const ESpecialTaskCategory cat) : TBase(base, "wp_category", ::ToString(cat)) , Category(cat) , WaitingHistogram(TBase::GetHistogram("Waiting/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) + , NoTasks(TBase::GetDeriviative("NoTasks")) , TaskExecuteHistogram(TBase::GetHistogram("TaskExecute/Duration/Us", NMonitoring::ExponentialHistogram(25, 2, 50))) , ValueWeight(TBase::GetValue("Weight")) , ExecuteDuration(TBase::GetDeriviative("Execute/Duration/Us")) { diff --git a/ydb/core/tx/conveyor_composite/service/counters.h b/ydb/core/tx/conveyor_composite/service/counters.h index 5c21e77af2ab..3049079e91ef 100644 --- a/ydb/core/tx/conveyor_composite/service/counters.h +++ b/ydb/core/tx/conveyor_composite/service/counters.h @@ -36,6 +36,7 @@ class TWPCategorySignals: public NColumnShard::TCommonCountersOwner { public: const ::NMonitoring::THistogramPtr WaitingHistogram; + const ::NMonitoring::TDynamicCounters::TCounterPtr NoTasks; const ::NMonitoring::THistogramPtr TaskExecuteHistogram; const ::NMonitoring::TDynamicCounters::TCounterPtr ValueWeight; @@ -76,7 +77,6 @@ class TWorkersPoolCounters: public NColumnShard::TCommonCountersOwner { const ::NMonitoring::TDynamicCounters::TCounterPtr WaitWorkerRate; const ::NMonitoring::TDynamicCounters::TCounterPtr UseWorkerRate; const ::NMonitoring::TDynamicCounters::TCounterPtr ChangeCPULimitRate; - const ::NMonitoring::TDynamicCounters::TCounterPtr NoTasks; TWorkersPoolCounters(const TString& poolName, const NColumnShard::TCommonCountersOwner& owner); diff --git a/ydb/core/tx/conveyor_composite/service/manager.h b/ydb/core/tx/conveyor_composite/service/manager.h index f1184b265e73..3b929afd64c1 100644 --- a/ydb/core/tx/conveyor_composite/service/manager.h +++ b/ydb/core/tx/conveyor_composite/service/manager.h @@ -25,12 +25,6 @@ class TTasksManager { return sb; } - void DoQuant(const TMonotonic newStart) { - for (auto&& c : Categories) { - c->DoQuant(newStart); - } - } - TTasksManager(const TString& /*convName*/, const NConfig::TConfig& config, const NActors::TActorId distributorActorId, TCounters& counters) : DistributorActorId(distributorActorId) , Config(config) diff --git a/ydb/core/tx/conveyor_composite/service/process.h b/ydb/core/tx/conveyor_composite/service/process.h index d8ebe6a2d2dc..c37f3fb48eed 100644 --- a/ydb/core/tx/conveyor_composite/service/process.h +++ b/ydb/core/tx/conveyor_composite/service/process.h @@ -74,16 +74,29 @@ class TProcess: public TNonCopyable, public NColumnShard::TMonitoringObjectsCoun YDB_READONLY_DEF(std::shared_ptr, Scope); std::shared_ptr WaitingTasksCount; + TPositiveControlInteger InProgressTasksCount; TAverageCalcer AverageTaskDuration; ui32 LinksCount = 0; + TDuration BaseWeight = TDuration::Zero(); public: - ~TProcess() { - WaitingTasksCount->Sub(Tasks.size()); + ui32 GetInProgressTasksCount() const { + return InProgressTasksCount.Val(); + } + + void SetBaseWeight(const TDuration d) { + BaseWeight = d; + CPUUsage->Clear(); + AFL_VERIFY(InProgressTasksCount.Val() == 0); + AFL_VERIFY(Tasks.size() == 0); } - void DoQuant(const TMonotonic newStart) { - CPUUsage->Cut(newStart); + TDuration GetWeightedUsage() const { + return BaseWeight + CPUUsage->CalcWeight(GetWeight()); + } + + ~TProcess() { + WaitingTasksCount->Sub(Tasks.size()); } bool HasTasks() const { @@ -98,12 +111,14 @@ class TProcess: public TNonCopyable, public NColumnShard::TMonitoringObjectsCoun auto result = Tasks.pop(); CPUUsage->AddPredicted(result.GetPredictedDuration()); WaitingTasksCount->Dec(); + InProgressTasksCount.Inc(); return std::move(result).BuildTask(signals->GetTaskSignals(result.GetTask()->GetTaskClassIdentifier())); } void PutTaskResult(TWorkerTaskResult&& result) { CPUUsage->Exchange(result.GetPredictedDuration(), result.GetStart(), result.GetFinish()); AverageTaskDuration.Add(result.GetDuration()); + InProgressTasksCount.Dec(); } [[nodiscard]] bool DecRegistration() { diff --git a/ydb/core/tx/conveyor_composite/service/service.cpp b/ydb/core/tx/conveyor_composite/service/service.cpp index 8d68579810a0..2900d76bd8ff 100644 --- a/ydb/core/tx/conveyor_composite/service/service.cpp +++ b/ydb/core/tx/conveyor_composite/service/service.cpp @@ -23,13 +23,6 @@ void TDistributor::Bootstrap() { TBase::Schedule(TDuration::Seconds(1), new NActors::TEvents::TEvWakeup(1)); } -void TDistributor::HandleMain(NActors::TEvents::TEvWakeup::TPtr& evExt) { - if (evExt->Get()->Tag == 1) { - Manager->DoQuant(TMonotonic::Now()); - TBase::Schedule(TDuration::Seconds(1), new NActors::TEvents::TEvWakeup(1)); - } -} - void TDistributor::HandleMain(TEvInternal::TEvTaskProcessedResult::TPtr& evExt) { auto& ev = *evExt->Get(); const TDuration backSendDuration = (TMonotonic::Now() - ev.GetConstructInstant()); diff --git a/ydb/core/tx/conveyor_composite/service/service.h b/ydb/core/tx/conveyor_composite/service/service.h index 4794c9a91bae..1dad336e0830 100644 --- a/ydb/core/tx/conveyor_composite/service/service.h +++ b/ydb/core/tx/conveyor_composite/service/service.h @@ -26,7 +26,6 @@ class TDistributor: public TActorBootstrapped { TMonotonic LastAddProcessInstant = TMonotonic::Now(); void HandleMain(TEvExecution::TEvNewTask::TPtr& ev); - void HandleMain(NActors::TEvents::TEvWakeup::TPtr& ev); void HandleMain(TEvExecution::TEvRegisterProcess::TPtr& ev); void HandleMain(TEvExecution::TEvUnregisterProcess::TPtr& ev); void HandleMain(TEvInternal::TEvTaskProcessedResult::TPtr& ev); @@ -47,7 +46,6 @@ class TDistributor: public TActorBootstrapped { // ("workers", Workers.size())("waiting", Waiting.size())("actor_id", SelfId()); switch (ev->GetTypeRewrite()) { hFunc(TEvExecution::TEvNewTask, HandleMain); - hFunc(NActors::TEvents::TEvWakeup, HandleMain); hFunc(TEvInternal::TEvTaskProcessedResult, HandleMain); hFunc(TEvExecution::TEvRegisterProcess, HandleMain); hFunc(TEvExecution::TEvUnregisterProcess, HandleMain); diff --git a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp index d7f36ce15db1..b185b6be9d3d 100644 --- a/ydb/core/tx/conveyor_composite/service/workers_pool.cpp +++ b/ydb/core/tx/conveyor_composite/service/workers_pool.cpp @@ -89,8 +89,10 @@ bool TWorkersPool::DrainTasks() { RunTask(std::move(tasks)); } } - if (!Processes.front().GetCategory()->HasTasks()) { - Counters->NoTasks->Add(1); + for (auto&& i : Processes) { + if (!i.GetCategory()->HasTasks()) { + i.GetCounters()->NoTasks->Add(1); + } } return newTask; } From 47f09bc4aa9d223ff5b1a74c0d9679acca03342a Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 19 Jun 2025 22:03:37 +0300 Subject: [PATCH 28/62] fix deadlock on special config (#19912) --- ydb/core/tx/columnshard/data_accessor/manager.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/columnshard/data_accessor/manager.cpp b/ydb/core/tx/columnshard/data_accessor/manager.cpp index 72d1244fa6f9..06d77a368c90 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.cpp +++ b/ydb/core/tx/columnshard/data_accessor/manager.cpp @@ -10,10 +10,12 @@ void TLocalManager::DrainQueue() { std::optional lastPathId; IGranuleDataAccessor* lastDataAccessor = nullptr; TPositiveControlInteger countToFlight; - while (PortionsAskInFlight + countToFlight < NYDBTest::TControllers::GetColumnShardController()->GetLimitForPortionsMetadataAsk() && - PortionsAsk.size()) { + const ui32 inFlightLimit = NYDBTest::TControllers::GetColumnShardController()->GetLimitForPortionsMetadataAsk(); + while (PortionsAskInFlight + countToFlight < inFlightLimit && PortionsAsk.size()) { THashMap portionsToAsk; - while (PortionsAskInFlight + countToFlight < 1000 && PortionsAsk.size()) { + ui32 packPortionsCount = 0; + while (PortionsAskInFlight + countToFlight < inFlightLimit && packPortionsCount < std::min(inFlightLimit, 1000) && + PortionsAsk.size()) { auto p = PortionsAsk.front().ExtractPortion(); const TString consumerId = PortionsAsk.front().GetConsumerId(); PortionsAsk.pop_front(); @@ -48,6 +50,7 @@ void TLocalManager::DrainQueue() { RequestsByPortion.erase(it); } else { portionsToAsk[p->GetPathId()].UpsertConsumer(consumerId).AddPortion(p); + ++packPortionsCount; ++countToFlight; } } From cad1adc3bdde98fc811ea7a252d711773dbba1ca Mon Sep 17 00:00:00 2001 From: xyliganSereja Date: Fri, 20 Jun 2025 09:45:14 +0300 Subject: [PATCH 29/62] move slide_limiter to library #19682 (#19906) --- ydb/core/driver_lib/run/kikimr_services_initializers.cpp | 6 +++--- ydb/core/driver_lib/run/ya.make | 2 +- ydb/core/tx/columnshard/columnshard__write_index.cpp | 5 ++--- ydb/core/tx/limiter/ya.make | 2 -- .../limiter => library/slide_limiter}/service/service.cpp | 0 .../tx/limiter => library/slide_limiter}/service/service.h | 6 +++--- .../tx/limiter => library/slide_limiter}/service/ya.make | 2 +- .../tx/limiter => library/slide_limiter}/usage/abstract.cpp | 0 .../tx/limiter => library/slide_limiter}/usage/abstract.h | 0 .../tx/limiter => library/slide_limiter}/usage/config.cpp | 0 .../tx/limiter => library/slide_limiter}/usage/config.h | 0 .../tx/limiter => library/slide_limiter}/usage/events.cpp | 0 .../tx/limiter => library/slide_limiter}/usage/events.h | 0 .../tx/limiter => library/slide_limiter}/usage/service.cpp | 0 .../tx/limiter => library/slide_limiter}/usage/service.h | 5 ++--- .../tx/limiter => library/slide_limiter}/usage/ya.make | 0 ydb/library/slide_limiter/ya.make | 4 ++++ ydb/library/ya.make | 1 + 18 files changed, 17 insertions(+), 16 deletions(-) rename ydb/{core/tx/limiter => library/slide_limiter}/service/service.cpp (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/service/service.h (95%) rename ydb/{core/tx/limiter => library/slide_limiter}/service/ya.make (67%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/abstract.cpp (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/abstract.h (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/config.cpp (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/config.h (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/events.cpp (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/events.h (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/service.cpp (100%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/service.h (94%) rename ydb/{core/tx/limiter => library/slide_limiter}/usage/ya.make (100%) create mode 100644 ydb/library/slide_limiter/ya.make diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 9130c57e319c..efd299c1520d 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -191,9 +191,6 @@ #include #include #include -#include -#include -#include #include #include @@ -233,6 +230,9 @@ #include #include #include +#include +#include +#include #include #include diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index 5a14be62419c..b5cf427c036e 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -109,7 +109,6 @@ PEERDIR( ydb/core/tx/coordinator ydb/core/tx/datashard ydb/core/tx/limiter/grouped_memory/usage - ydb/core/tx/limiter/service ydb/core/tx/long_tx_service ydb/core/tx/long_tx_service/public ydb/core/tx/mediator @@ -142,6 +141,7 @@ PEERDIR( ydb/library/security ydb/library/signal_backtrace ydb/library/yql/providers/pq/cm_client + ydb/library/slide_limiter/service ydb/library/yql/providers/s3/actors ydb/public/lib/base ydb/public/lib/deprecated/client diff --git a/ydb/core/tx/columnshard/columnshard__write_index.cpp b/ydb/core/tx/columnshard/columnshard__write_index.cpp index 87cdc9909d34..d2d568bb3b0f 100644 --- a/ydb/core/tx/columnshard/columnshard__write_index.cpp +++ b/ydb/core/tx/columnshard/columnshard__write_index.cpp @@ -6,10 +6,9 @@ #include "engines/changes/abstract/abstract.h" #include "engines/writer/compacted_blob_constructor.h" -#include -#include - #include +#include +#include namespace NKikimr::NColumnShard { diff --git a/ydb/core/tx/limiter/ya.make b/ydb/core/tx/limiter/ya.make index c27ad41aa02b..a861bfe9ea97 100644 --- a/ydb/core/tx/limiter/ya.make +++ b/ydb/core/tx/limiter/ya.make @@ -1,5 +1,3 @@ RECURSE( grouped_memory - service - usage ) diff --git a/ydb/core/tx/limiter/service/service.cpp b/ydb/library/slide_limiter/service/service.cpp similarity index 100% rename from ydb/core/tx/limiter/service/service.cpp rename to ydb/library/slide_limiter/service/service.cpp diff --git a/ydb/core/tx/limiter/service/service.h b/ydb/library/slide_limiter/service/service.h similarity index 95% rename from ydb/core/tx/limiter/service/service.h rename to ydb/library/slide_limiter/service/service.h index 4082379e2086..921ae7972d44 100644 --- a/ydb/core/tx/limiter/service/service.h +++ b/ydb/library/slide_limiter/service/service.h @@ -1,7 +1,7 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include diff --git a/ydb/core/tx/limiter/service/ya.make b/ydb/library/slide_limiter/service/ya.make similarity index 67% rename from ydb/core/tx/limiter/service/ya.make rename to ydb/library/slide_limiter/service/ya.make index 5560098a6b5a..7d770b9f3155 100644 --- a/ydb/core/tx/limiter/service/ya.make +++ b/ydb/library/slide_limiter/service/ya.make @@ -5,7 +5,7 @@ SRCS( ) PEERDIR( - ydb/core/tx/limiter/usage + ydb/library/slide_limiter/usage ydb/core/protos ) diff --git a/ydb/core/tx/limiter/usage/abstract.cpp b/ydb/library/slide_limiter/usage/abstract.cpp similarity index 100% rename from ydb/core/tx/limiter/usage/abstract.cpp rename to ydb/library/slide_limiter/usage/abstract.cpp diff --git a/ydb/core/tx/limiter/usage/abstract.h b/ydb/library/slide_limiter/usage/abstract.h similarity index 100% rename from ydb/core/tx/limiter/usage/abstract.h rename to ydb/library/slide_limiter/usage/abstract.h diff --git a/ydb/core/tx/limiter/usage/config.cpp b/ydb/library/slide_limiter/usage/config.cpp similarity index 100% rename from ydb/core/tx/limiter/usage/config.cpp rename to ydb/library/slide_limiter/usage/config.cpp diff --git a/ydb/core/tx/limiter/usage/config.h b/ydb/library/slide_limiter/usage/config.h similarity index 100% rename from ydb/core/tx/limiter/usage/config.h rename to ydb/library/slide_limiter/usage/config.h diff --git a/ydb/core/tx/limiter/usage/events.cpp b/ydb/library/slide_limiter/usage/events.cpp similarity index 100% rename from ydb/core/tx/limiter/usage/events.cpp rename to ydb/library/slide_limiter/usage/events.cpp diff --git a/ydb/core/tx/limiter/usage/events.h b/ydb/library/slide_limiter/usage/events.h similarity index 100% rename from ydb/core/tx/limiter/usage/events.h rename to ydb/library/slide_limiter/usage/events.h diff --git a/ydb/core/tx/limiter/usage/service.cpp b/ydb/library/slide_limiter/usage/service.cpp similarity index 100% rename from ydb/core/tx/limiter/usage/service.cpp rename to ydb/library/slide_limiter/usage/service.cpp diff --git a/ydb/core/tx/limiter/usage/service.h b/ydb/library/slide_limiter/usage/service.h similarity index 94% rename from ydb/core/tx/limiter/usage/service.h rename to ydb/library/slide_limiter/usage/service.h index 7ac3946e3995..c7ce2b2a752e 100644 --- a/ydb/core/tx/limiter/usage/service.h +++ b/ydb/library/slide_limiter/usage/service.h @@ -1,11 +1,10 @@ #pragma once #include "config.h" -#include -#include - #include #include +#include +#include namespace NKikimr::NLimiter { diff --git a/ydb/core/tx/limiter/usage/ya.make b/ydb/library/slide_limiter/usage/ya.make similarity index 100% rename from ydb/core/tx/limiter/usage/ya.make rename to ydb/library/slide_limiter/usage/ya.make diff --git a/ydb/library/slide_limiter/ya.make b/ydb/library/slide_limiter/ya.make new file mode 100644 index 000000000000..f0fb98f06961 --- /dev/null +++ b/ydb/library/slide_limiter/ya.make @@ -0,0 +1,4 @@ +RECURSE( + service + usage +) \ No newline at end of file diff --git a/ydb/library/ya.make b/ydb/library/ya.make index 8b2c71675531..95aea84ff6fa 100644 --- a/ydb/library/ya.make +++ b/ydb/library/ya.make @@ -31,6 +31,7 @@ RECURSE( security signal_backtrace signals + slide_limiter table_creator testlib time_series_vec From 2764561a7f40ab79efecf59e434406e14cbe49af Mon Sep 17 00:00:00 2001 From: Vladilen Date: Fri, 20 Jun 2025 18:26:33 +0200 Subject: [PATCH 30/62] Add CS compaction ResourceBroker queues configuration to MemoryController (#19708) (#19817) --- ydb/core/base/localdb.cpp | 4 ++ ydb/core/base/localdb.h | 4 ++ .../memory_controller/memory_controller.cpp | 63 ++++++++++------ .../memory_controller/memory_controller.h | 2 + .../memory_controller_config.h | 5 +- .../memory_controller_ut.cpp | 72 +++++++++++++++---- .../protos/memory_controller_config.proto | 7 ++ ydb/core/tablet/resource_broker.cpp | 16 ++--- ydb/core/tx/columnshard/common/limits.h | 14 ++++ 9 files changed, 145 insertions(+), 42 deletions(-) diff --git a/ydb/core/base/localdb.cpp b/ydb/core/base/localdb.cpp index 8a35d11e8b0c..4786daa0c5ce 100644 --- a/ydb/core/base/localdb.cpp +++ b/ydb/core/base/localdb.cpp @@ -262,4 +262,8 @@ const TString KqpResourceManagerTaskName = "kqp_query"; const TString KqpResourceManagerQueue = "queue_kqp_resource_manager"; const TString LegacyQueueIdTaskNamePrefix = "compaction_gen"; +const TString ColumnShardCompactionIndexationQueue = "queue_cs_indexation"; +const TString ColumnShardCompactionTtlQueue = "queue_cs_ttl"; +const TString ColumnShardCompactionGeneralQueue = "queue_cs_general"; +const TString ColumnShardCompactionNormalizerQueue = "queue_cs_normalizer"; }} diff --git a/ydb/core/base/localdb.h b/ydb/core/base/localdb.h index eec65265b072..695ed72d94b9 100644 --- a/ydb/core/base/localdb.h +++ b/ydb/core/base/localdb.h @@ -152,4 +152,8 @@ extern const TString KqpResourceManagerTaskName; extern const TString KqpResourceManagerQueue; extern const TString LegacyQueueIdTaskNamePrefix; +extern const TString ColumnShardCompactionIndexationQueue; +extern const TString ColumnShardCompactionTtlQueue; +extern const TString ColumnShardCompactionGeneralQueue; +extern const TString ColumnShardCompactionNormalizerQueue; }} diff --git a/ydb/core/memory_controller/memory_controller.cpp b/ydb/core/memory_controller/memory_controller.cpp index 1d17c67c0684..9dd462422b11 100644 --- a/ydb/core/memory_controller/memory_controller.cpp +++ b/ydb/core/memory_controller/memory_controller.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -285,22 +286,7 @@ class TMemoryController : public TActorBootstrapped { Counters->GetCounter("Stats/ConsumersLimit")->Set(consumersLimitBytes); - ui64 queryExecutionConsumption = TAlignedPagePool::GetGlobalPagePoolSize(); - ui64 queryExecutionLimitBytes = ResourceBrokerSelfConfig.QueryExecutionLimitBytes - ? ResourceBrokerSelfConfig.QueryExecutionLimitBytes // for backward compatibility - : GetQueryExecutionLimitBytes(Config, hardLimitBytes); - LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Consumer QueryExecution state:" - << " Consumption: " << HumanReadableBytes(queryExecutionConsumption) << " Limit: " << HumanReadableBytes(queryExecutionLimitBytes)); - Counters->GetCounter("Consumer/QueryExecution/Consumption")->Set(queryExecutionConsumption); - Counters->GetCounter("Consumer/QueryExecution/Limit")->Set(queryExecutionLimitBytes); - memoryStats.SetQueryExecutionConsumption(queryExecutionConsumption); - memoryStats.SetQueryExecutionLimit(queryExecutionLimitBytes); - - // Note: for now ResourceBroker and its queues aren't MemoryController consumers and don't share limits with other caches - ApplyResourceBrokerConfig({ - activitiesLimitBytes, - queryExecutionLimitBytes - }); + ProcessResourceBrokerConfig(ctx, memoryStats, hardLimitBytes, activitiesLimitBytes); Send(NNodeWhiteboard::MakeNodeWhiteboardServiceId(SelfId().NodeId()), memoryStatsUpdate); @@ -387,6 +373,27 @@ class TMemoryController : public TActorBootstrapped { } } + void ProcessResourceBrokerConfig(const TActorContext& ctx, NKikimrMemory::TMemoryStats& memoryStats, ui64 hardLimitBytes, + ui64 activitiesLimitBytes) { + ui64 queryExecutionConsumption = TAlignedPagePool::GetGlobalPagePoolSize(); + ui64 queryExecutionLimitBytes = ResourceBrokerSelfConfig.QueryExecutionLimitBytes + ? ResourceBrokerSelfConfig.QueryExecutionLimitBytes // for backward compatibility + : GetQueryExecutionLimitBytes(Config, hardLimitBytes); + LOG_INFO_S(ctx, NKikimrServices::MEMORY_CONTROLLER, "Consumer QueryExecution state:" << " Consumption: " << HumanReadableBytes(queryExecutionConsumption) << " Limit: " << HumanReadableBytes(queryExecutionLimitBytes)); + Counters->GetCounter("Consumer/QueryExecution/Consumption")->Set(queryExecutionConsumption); + Counters->GetCounter("Consumer/QueryExecution/Limit")->Set(queryExecutionLimitBytes); + memoryStats.SetQueryExecutionConsumption(queryExecutionConsumption); + memoryStats.SetQueryExecutionLimit(queryExecutionLimitBytes); + + ui64 columnTablesCompactionLimitBytes = GetColumnTablesCompactionLimitBytes(Config, hardLimitBytes); + + // Note: for now ResourceBroker and its queues aren't MemoryController consumers and don't share limits with other caches + ApplyResourceBrokerConfig({ + .LimitBytes = activitiesLimitBytes, + .QueryExecutionLimitBytes = queryExecutionLimitBytes, + .ColumnTablesCompactionLimitBytes = columnTablesCompactionLimitBytes}); + } + void ApplyResourceBrokerConfig(TResourceBrokerConfig config) { if (config == CurrentResourceBrokerConfig) { return; @@ -394,17 +401,33 @@ class TMemoryController : public TActorBootstrapped { TAutoPtr configure = new TEvResourceBroker::TEvConfigure(); configure->Merge = true; - configure->Record.MutableResourceLimit()->SetMemory(config.LimitBytes); - auto queue = configure->Record.AddQueues(); - queue->SetName(NLocalDb::KqpResourceManagerQueue); - queue->MutableLimit()->SetMemory(config.QueryExecutionLimitBytes); + auto& record = configure->Record; + record.MutableResourceLimit()->SetMemory(config.LimitBytes); + + // Compaction uses 4 queues, but there is only one memory limit setting in the configuration, + // so the coefficients are used to split allocated memory between the queues. + AddLimitToQueueConfig(record, NLocalDb::KqpResourceManagerQueue, config.QueryExecutionLimitBytes); + AddLimitToQueueConfig(record, NLocalDb::ColumnShardCompactionIndexationQueue, + config.ColumnTablesCompactionLimitBytes * NKikimr::NOlap::TGlobalLimits::CompactionIndexationQueueLimitCoefficient); + AddLimitToQueueConfig(record, NLocalDb::ColumnShardCompactionTtlQueue, + config.ColumnTablesCompactionLimitBytes * NKikimr::NOlap::TGlobalLimits::CompactionTtlQueueLimitCoefficient); + AddLimitToQueueConfig(record, NLocalDb::ColumnShardCompactionGeneralQueue, + config.ColumnTablesCompactionLimitBytes * NKikimr::NOlap::TGlobalLimits::CompactionGeneralQueueLimitCoefficient); + AddLimitToQueueConfig(record, NLocalDb::ColumnShardCompactionNormalizerQueue, + config.ColumnTablesCompactionLimitBytes * NKikimr::NOlap::TGlobalLimits::CompactionNormalizerQueueLimitCoefficient); Send(MakeResourceBrokerID(), configure.Release()); CurrentResourceBrokerConfig.emplace(std::move(config)); } + void AddLimitToQueueConfig(NKikimrResourceBroker::TResourceBrokerConfig& record, const TString& name, const ui64 limitBytes) { + auto queue = record.AddQueues(); + queue->SetName(name); + queue->MutableLimit()->SetMemory(limitBytes); + } + TConsumerCounters& GetConsumerCounters(EMemoryConsumerKind consumer) { auto it = ConsumerCounters.FindPtr(consumer); if (it) { diff --git a/ydb/core/memory_controller/memory_controller.h b/ydb/core/memory_controller/memory_controller.h index f3428d173997..21a23e5a327c 100644 --- a/ydb/core/memory_controller/memory_controller.h +++ b/ydb/core/memory_controller/memory_controller.h @@ -13,6 +13,7 @@ namespace NKikimr::NMemory { struct TResourceBrokerConfig { ui64 LimitBytes = 0; ui64 QueryExecutionLimitBytes = 0; + ui64 ColumnTablesCompactionLimitBytes = 0; auto operator<=>(const TResourceBrokerConfig&) const = default; @@ -20,6 +21,7 @@ struct TResourceBrokerConfig { TStringBuilder result; result << "LimitBytes: " << LimitBytes; result << " QueryExecutionLimitBytes: " << QueryExecutionLimitBytes; + result << " ColumnTablesCompactionLimitBytes: " << ColumnTablesCompactionLimitBytes; return result; } }; diff --git a/ydb/core/memory_controller/memory_controller_config.h b/ydb/core/memory_controller/memory_controller_config.h index cdbd85f41873..81d482235472 100644 --- a/ydb/core/memory_controller/memory_controller_config.h +++ b/ydb/core/memory_controller/memory_controller_config.h @@ -7,7 +7,7 @@ namespace NKikimr::NMemory { namespace { - + ui64 GetPercent(float percent, ui64 value) { return static_cast(static_cast(value) * (percent / 100.0)); } @@ -80,4 +80,7 @@ GET_MAX_LIMIT(SharedCache) GET_LIMIT(QueryExecutionLimit) +GET_LIMIT(ColumnTablesReadExecutionLimit) +GET_LIMIT(ColumnTablesCompactionLimit) +GET_LIMIT(ColumnTablesCacheLimit) } \ No newline at end of file diff --git a/ydb/core/memory_controller/memory_controller_ut.cpp b/ydb/core/memory_controller/memory_controller_ut.cpp index b707f7af8ee0..0ce1ceedec27 100644 --- a/ydb/core/memory_controller/memory_controller_ut.cpp +++ b/ydb/core/memory_controller/memory_controller_ut.cpp @@ -403,7 +403,7 @@ Y_UNIT_TEST(ResourceBroker) { auto resourceBrokerConfig = serverSettings.AppConfig->MutableResourceBrokerConfig(); auto queue = resourceBrokerConfig->AddQueues(); - queue->SetName("queue_cs_ttl"); + queue->SetName("queue_cs_scan_read"); queue->MutableLimit()->SetMemory(13_MB); auto server = MakeIntrusive(serverSettings); @@ -439,14 +439,10 @@ Y_UNIT_TEST(ResourceBroker) { UNIT_ASSERT_VALUES_EQUAL(config->Get()->QueueConfig->GetLimit().GetMemory(), 75_MB); // ensure that other settings are not affected: - runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_ttl"))); + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_scan_read"))); config = runtime.GrabEdgeEvent(sender); UNIT_ASSERT_VALUES_EQUAL(config->Get()->QueueConfig->GetLimit().GetCpu(), 3); UNIT_ASSERT_VALUES_EQUAL(config->Get()->QueueConfig->GetLimit().GetMemory(), 13_MB); - runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_general"))); - config = runtime.GrabEdgeEvent(sender); - UNIT_ASSERT_VALUES_EQUAL(config->Get()->QueueConfig->GetLimit().GetCpu(), 3); - UNIT_ASSERT_VALUES_EQUAL(config->Get()->QueueConfig->GetLimit().GetMemory(), 3221225472); } Y_UNIT_TEST(ResourceBroker_ConfigLimit) { @@ -463,10 +459,10 @@ Y_UNIT_TEST(ResourceBroker_ConfigLimit) { auto resourceBrokerConfig = serverSettings.AppConfig->MutableResourceBrokerConfig(); resourceBrokerConfig->MutableResourceLimit()->SetMemory(1000_MB); auto queue = resourceBrokerConfig->AddQueues(); - queue->SetName("queue_kqp_resource_manager"); + queue->SetName(NLocalDb::KqpResourceManagerQueue); queue->MutableLimit()->SetMemory(999_MB); queue = resourceBrokerConfig->AddQueues(); - queue->SetName("queue_cs_ttl"); + queue->SetName("queue_cs_scan_read"); queue->MutableLimit()->SetMemory(13_MB); auto server = MakeIntrusive(serverSettings); @@ -493,16 +489,66 @@ Y_UNIT_TEST(ResourceBroker_ConfigLimit) { UNIT_ASSERT_VALUES_EQUAL(server->MemoryControllerCounters->GetCounter("Stats/ActivitiesLimitBytes")->Val(), 1000_MB); // ensure that other settings are not affected: - runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_ttl"))); + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_scan_read"))); config = runtime.GrabEdgeEvent(handle); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetCpu(), 3); UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 13_MB); - runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest("queue_cs_general"))); - config = runtime.GrabEdgeEvent(handle); - UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetCpu(), 3); - UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), 3221225472); } +Y_UNIT_TEST(ResourceBroker_ConfigCS) { + using namespace NResourceBroker; + + TPortManager pm; + TServerSettings serverSettings(pm.GetPort(2134)); + serverSettings.SetDomainName("Root").SetUseRealThreads(false); + + auto memoryControllerConfig = serverSettings.AppConfig->MutableMemoryControllerConfig(); + memoryControllerConfig->SetColumnTablesCompactionLimitPercent(32); + + auto resourceBrokerConfig = serverSettings.AppConfig->MutableResourceBrokerConfig(); + resourceBrokerConfig->MutableResourceLimit()->SetMemory(1000_MB); + + auto addQueueWithMemoryLimit = [&](const TString& name, const ui64 memoryLimit) { + auto queue = resourceBrokerConfig->AddQueues(); + queue->SetName(name); + queue->MutableLimit()->SetMemory(memoryLimit); + }; + + addQueueWithMemoryLimit(NLocalDb::ColumnShardCompactionIndexationQueue, 1_MB); + addQueueWithMemoryLimit(NLocalDb::ColumnShardCompactionTtlQueue, 1_MB); + addQueueWithMemoryLimit(NLocalDb::ColumnShardCompactionGeneralQueue, 1_MB); + addQueueWithMemoryLimit(NLocalDb::ColumnShardCompactionNormalizerQueue, 1_MB); + + auto server = MakeIntrusive(serverSettings); + server->ProcessMemoryInfo->CGroupLimit = 1000_MB; + auto& runtime = *server->GetRuntime(); + TAutoPtr handle; + auto sender = runtime.AllocateEdgeActor(); + + InitRoot(server, sender); + + runtime.SimulateSleep(TDuration::Seconds(2)); + + auto checkMemoryLimit = [&](const TString& queueName, const ui64 expectedLimit) { + runtime.Send(new IEventHandle(MakeResourceBrokerID(), sender, new TEvResourceBroker::TEvConfigRequest(queueName))); + auto config = runtime.GrabEdgeEvent(handle); + UNIT_ASSERT_VALUES_EQUAL(config->QueueConfig->GetLimit().GetMemory(), expectedLimit); + }; + + checkMemoryLimit(NLocalDb::ColumnShardCompactionIndexationQueue, 40_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionTtlQueue, 40_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionGeneralQueue, 120_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionNormalizerQueue, 120_MB); + + // Check memory change + server->ProcessMemoryInfo->CGroupLimit = 50_MB; + runtime.SimulateSleep(TDuration::Seconds(2)); + + checkMemoryLimit(NLocalDb::ColumnShardCompactionIndexationQueue, 2_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionTtlQueue, 2_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionGeneralQueue, 6_MB); + checkMemoryLimit(NLocalDb::ColumnShardCompactionNormalizerQueue, 6_MB); +} } } diff --git a/ydb/core/protos/memory_controller_config.proto b/ydb/core/protos/memory_controller_config.proto index a4cd0ce337a3..2725c37f1723 100644 --- a/ydb/core/protos/memory_controller_config.proto +++ b/ydb/core/protos/memory_controller_config.proto @@ -25,4 +25,11 @@ message TMemoryControllerConfig { optional float QueryExecutionLimitPercent = 120 [default = 20]; optional uint64 QueryExecutionLimitBytes = 121; + + optional float ColumnTablesReadExecutionLimitPercent = 200 [default = 20]; + optional uint64 ColumnTablesReadExecutionLimitBytes = 201; + optional float ColumnTablesCompactionLimitPercent = 202 [default = 16]; + optional uint64 ColumnTablesCompactionLimitBytes = 203; + optional float ColumnTablesCacheLimitPercent = 204 [default = 4]; + optional uint64 ColumnTablesCacheLimitBytes = 205; } diff --git a/ydb/core/tablet/resource_broker.cpp b/ydb/core/tablet/resource_broker.cpp index 29cb3dc06072..4c6f3163d9ff 100644 --- a/ydb/core/tablet/resource_broker.cpp +++ b/ydb/core/tablet/resource_broker.cpp @@ -1345,19 +1345,19 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() queue->MutableLimit()->SetCpu(3); queue = config.AddQueues(); - queue->SetName("queue_cs_indexation"); + queue->SetName(NLocalDb::ColumnShardCompactionIndexationQueue); queue->SetWeight(100); queue->MutableLimit()->SetCpu(3); queue->MutableLimit()->SetMemory(CSInsertCompactionMemoryLimit); queue = config.AddQueues(); - queue->SetName("queue_cs_ttl"); + queue->SetName(NLocalDb::ColumnShardCompactionTtlQueue); queue->SetWeight(100); queue->MutableLimit()->SetCpu(3); queue->MutableLimit()->SetMemory(CSTTLCompactionMemoryLimit); queue = config.AddQueues(); - queue->SetName("queue_cs_general"); + queue->SetName(NLocalDb::ColumnShardCompactionGeneralQueue); queue->SetWeight(100); queue->MutableLimit()->SetCpu(3); queue->MutableLimit()->SetMemory(CSGeneralCompactionMemoryLimit); @@ -1369,7 +1369,7 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() queue->MutableLimit()->SetMemory(CSScanMemoryLimit); queue = config.AddQueues(); - queue->SetName("queue_cs_normalizer"); + queue->SetName(NLocalDb::ColumnShardCompactionNormalizerQueue); queue->SetWeight(100); queue->MutableLimit()->SetCpu(3); queue->MutableLimit()->SetMemory(CSScanMemoryLimit); @@ -1462,17 +1462,17 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() task = config.AddTasks(); task->SetName("CS::TTL"); - task->SetQueueName("queue_cs_ttl"); + task->SetQueueName(NLocalDb::ColumnShardCompactionTtlQueue); task->SetDefaultDuration(TDuration::Minutes(10).GetValue()); task = config.AddTasks(); task->SetName("CS::INDEXATION"); - task->SetQueueName("queue_cs_indexation"); + task->SetQueueName(NLocalDb::ColumnShardCompactionIndexationQueue); task->SetDefaultDuration(TDuration::Minutes(10).GetValue()); task = config.AddTasks(); task->SetName("CS::GENERAL"); - task->SetQueueName("queue_cs_general"); + task->SetQueueName(NLocalDb::ColumnShardCompactionGeneralQueue); task->SetDefaultDuration(TDuration::Minutes(10).GetValue()); task = config.AddTasks(); @@ -1482,7 +1482,7 @@ NKikimrResourceBroker::TResourceBrokerConfig MakeDefaultConfig() task = config.AddTasks(); task->SetName("CS::NORMALIZER"); - task->SetQueueName("queue_cs_normalizer"); + task->SetQueueName(NLocalDb::ColumnShardCompactionNormalizerQueue); task->SetDefaultDuration(TDuration::Minutes(10).GetValue()); task = config.AddTasks(); diff --git a/ydb/core/tx/columnshard/common/limits.h b/ydb/core/tx/columnshard/common/limits.h index e3869f26727f..8e93073d6554 100644 --- a/ydb/core/tx/columnshard/common/limits.h +++ b/ydb/core/tx/columnshard/common/limits.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace NKikimr::NOlap { class TGlobalLimits { @@ -11,5 +12,18 @@ class TGlobalLimits { static constexpr inline ui64 ScanMemoryLimit = 3ULL << 30; static constexpr inline ui64 DefaultReadSequentiallyBufferSize = ((ui64)8) << 20; + + static constexpr double CompactionIndexationQueueLimitCoefficient = 0.125; + static constexpr double CompactionTtlQueueLimitCoefficient = 0.125; + static constexpr double CompactionGeneralQueueLimitCoefficient = 0.375; + static constexpr double CompactionNormalizerQueueLimitCoefficient = 0.375; + + static_assert((CompactionIndexationQueueLimitCoefficient + CompactionTtlQueueLimitCoefficient + + CompactionGeneralQueueLimitCoefficient + CompactionNormalizerQueueLimitCoefficient - 1.0 < + std::numeric_limits::epsilon()) && + (1.0 - (CompactionIndexationQueueLimitCoefficient + CompactionTtlQueueLimitCoefficient + + CompactionGeneralQueueLimitCoefficient + CompactionNormalizerQueueLimitCoefficient) < + std::numeric_limits::epsilon()), + "Compaction coefficients sum must be equal to 1.0"); }; } \ No newline at end of file From 682c4ced39cde9cd1bb96404961d3163dd54ec8e Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Sat, 21 Jun 2025 12:17:20 +0300 Subject: [PATCH 31/62] Initial internal pathid based on TabletId (#19902) --- .../tx/columnshard/hooks/abstract/abstract.h | 4 ++++ .../tx/columnshard/hooks/testing/controller.h | 10 ++++++++++ .../tx/columnshard/hooks/testing/ro_controller.h | 4 ++++ ydb/core/tx/columnshard/tables_manager.cpp | 16 ++++++++++++++-- ydb/core/tx/columnshard/tables_manager.h | 3 +-- .../ut_schema/ut_columnshard_schema.cpp | 1 + 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index 490382372c7e..f147f8938b0f 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -357,6 +357,10 @@ class ICSController { return {}; } + virtual bool IsForcedGenerateInternalPathId() const { + return false; + } + virtual void OnAddPathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { } virtual void OnDeletePathId(const ui64 /* tabletId */, const NColumnShard::TUnifiedPathId& /* pathId */) { diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index a55a1447ecf5..d8e0351a004b 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -49,6 +49,8 @@ class TController: public TReadOnlyController { TMutex ActiveTabletsMutex; + bool ForcedGenerateInternalPathId = true; + using TInternalPathId = NKikimr::NColumnShard::TInternalPathId; using TSchemeShardLocalPathId = NKikimr::NColumnShard::TSchemeShardLocalPathId; using TUnifiedPathId = NKikimr::NColumnShard::TUnifiedPathId; @@ -400,6 +402,14 @@ class TController: public TReadOnlyController { (*tablet)->DeletePathId(pathId); } + virtual bool IsForcedGenerateInternalPathId() const override { + return ForcedGenerateInternalPathId; + } + + void SetForcedGenerateInternalPathId(const bool value) { + ForcedGenerateInternalPathId = value; + } + }; } diff --git a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h index 8646d3494051..2aadf6ec0011 100644 --- a/ydb/core/tx/columnshard/hooks/testing/ro_controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/ro_controller.h @@ -179,6 +179,10 @@ class TReadOnlyController: public ICSController { } } + virtual bool IsForcedGenerateInternalPathId() const override { + return true; + } + }; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 1c8e3383f5d7..bb158f95a601 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -18,6 +18,15 @@ void TSchemaPreset::Deserialize(const NKikimrSchemeOp::TColumnTableSchemaPreset& Name = presetProto.GetName(); } +namespace { + +TInternalPathId GetInitialMaxInternalPathId(const ui64 tabletId) { + static constexpr ui64 InternalPathIdBase = 1'000'000'000; //Use a value presumably greater than any really used + static constexpr ui64 InternalPathIdTabletMod = 1'000'000; //Use different start value for tablets + return TInternalPathId::FromRawValue(InternalPathIdBase + tabletId * InternalPathIdTabletMod); +} + +} //namespace std::optional TTablesManager::ResolveSchemeShardLocalPathId(const TInternalPathId internalPathId) const { if (!HasTable(internalPathId)) { @@ -71,6 +80,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { } if (maxPathId) { MaxInternalPathId =TInternalPathId::FromRawValue(*maxPathId); + AFL_VERIFY(MaxInternalPathId >= GetInitialMaxInternalPathId(TabletId)); } } { @@ -236,7 +246,8 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet } TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) { - if (AppData()->ColumnShardConfig.GetGenerateInternalPathId()) { + if (NYDBTest::TControllers::GetColumnShardController()->IsForcedGenerateInternalPathId() || + AppData()->ColumnShardConfig.GetGenerateInternalPathId()) { const auto result = TInternalPathId::FromRawValue(MaxInternalPathId.GetRawValue() + 1); MaxInternalPathId = result; return result; @@ -393,7 +404,8 @@ TTablesManager::TTablesManager(const std::shared_ptr& s , LoadTimeCounters(std::make_unique()) , SchemaObjectsCache(schemaCache) , PortionsStats(portionsStats) - , TabletId(tabletId) { + , TabletId(tabletId) + , MaxInternalPathId(GetInitialMaxInternalPathId(TabletId)) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const TInternalPathId pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 86b5e90fc9c1..4929f0dead3f 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -216,8 +216,7 @@ class TTablesManager: public NOlap::IPathIdTranslator { NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; ui64 TabletId = 0; - static constexpr ui64 MaxInternalPathIdDefault = 1'000'000'000; //Use a value presumably greater than any really used - TInternalPathId MaxInternalPathId = TInternalPathId::FromRawValue(MaxInternalPathIdDefault); //Max internal path id ever used in this tablet + TInternalPathId MaxInternalPathId; friend class TTxInit; diff --git a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp index cc232fa70911..b2ad32ab3ce7 100644 --- a/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp @@ -1061,6 +1061,7 @@ Y_UNIT_TEST_SUITE(TColumnShardTestSchema) { TTester::Setup(runtime); runtime.GetAppData().ColumnShardConfig.SetGenerateInternalPathId(generateInternalPathId); auto csDefaultControllerGuard = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard(); + csDefaultControllerGuard->SetForcedGenerateInternalPathId(generateInternalPathId); using namespace NTxUT; CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); From 6ab24e980c989cfa8edd8734b98b8471de6b45f3 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Sat, 21 Jun 2025 21:52:21 +0300 Subject: [PATCH 32/62] default portions count level limit (#19969) --- .../optimizer/lcbuckets/constructor/level/zero_level.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp index a1c09c28b5f0..2ce33e03a724 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp @@ -96,8 +96,9 @@ std::shared_ptr T const std::shared_ptr& nextLevel, const ui32 indexLevel, const std::shared_ptr& /*portionsInfo*/, const TLevelCounters& counters, const std::vector>& selectors) const { return std::make_shared(indexLevel, nextLevel, counters, - std::make_shared(PortionsCountLimit, PortionsSizeLimit), PortionsLiveDuration.value_or(TDuration::Max()), - ExpectedBlobsSize.value_or((ui64)1 << 20), PortionsCountAvailable.value_or(10), selectors, GetDefaultSelectorName()); + std::make_shared(PortionsCountLimit.value_or(1000000), PortionsSizeLimit), + PortionsLiveDuration.value_or(TDuration::Max()), ExpectedBlobsSize.value_or((ui64)1 << 20), PortionsCountAvailable.value_or(10), + selectors, GetDefaultSelectorName()); } } // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets From c82b9eea005b31cc2847b541360837790e2fb98c Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Mon, 23 Jun 2025 10:55:01 +0300 Subject: [PATCH 33/62] clarify TLockFeatures (#20017) --- .../constructor/read_metadata.cpp | 2 +- .../tx/columnshard/operations/manager.cpp | 9 ++-- ydb/core/tx/columnshard/operations/manager.h | 44 +++++++++++-------- .../tx/columnshard/tablet/write_queue.cpp | 2 +- 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp index 0f223dbde9ce..28987caa3a41 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/constructor/read_metadata.cpp @@ -105,7 +105,7 @@ void TReadMetadata::DoOnReplyConstruction(const ui64 tabletId, NKqp::NInternalIm lockInfo.SetLockId(LockSharingInfo->GetLockId()); lockInfo.SetGeneration(LockSharingInfo->GetGeneration()); lockInfo.SetDataShard(tabletId); - lockInfo.SetCounter(LockSharingInfo->GetCounter()); + lockInfo.SetCounter(LockSharingInfo->GetInternalGenerationCounter()); PathId.SchemeShardLocalPathId.ToProto(lockInfo); lockInfo.SetHasWrites(LockSharingInfo->HasWrites()); if (LockSharingInfo->IsBroken()) { diff --git a/ydb/core/tx/columnshard/operations/manager.cpp b/ydb/core/tx/columnshard/operations/manager.cpp index 6b0d6adc0c76..362a2386a249 100644 --- a/ydb/core/tx/columnshard/operations/manager.cpp +++ b/ydb/core/tx/columnshard/operations/manager.cpp @@ -40,7 +40,7 @@ bool TOperationsManager::Load(NTabletFlatExecutor::TTransactionContext& txc) { if (it == LockFeatures.end()) { it = LockFeatures.emplace(lockId, TLockFeatures(lockId, 0)).first; } - it->second.MutableWriteOperations().emplace_back(operation); + it->second.AddWriteOperation(operation); LastWriteId = std::max(LastWriteId, operation->GetWriteId()); if (!rowset.Next()) { return false; @@ -202,7 +202,7 @@ void TOperationsManager::LinkTransactionOnExecute(const ui64 lockId, const ui64 void TOperationsManager::LinkTransactionOnComplete(const ui64 /*lockId*/, const ui64 /*txId*/) { } -TWriteOperation::TPtr TOperationsManager::RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, +TWriteOperation::TPtr TOperationsManager::CreateWriteOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType) { auto writeId = BuildNextOperationWriteId(); auto operation = std::make_shared(pathId, writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), @@ -210,8 +210,7 @@ TWriteOperation::TPtr TOperationsManager::RegisterOperation(const TUnifiedPathId AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "register_operation")("operation_id", operation->GetWriteId())( "last", LastWriteId); AFL_VERIFY(Operations.emplace(operation->GetWriteId(), operation).second); - GetLockVerified(operation->GetLockId()).MutableWriteOperations().emplace_back(operation); - GetLockVerified(operation->GetLockId()).AddWrite(); + GetLockVerified(operation->GetLockId()).AddWriteOperation(operation); return operation; } @@ -289,7 +288,7 @@ void TOperationsManager::AddEventForLock( if (auto txEvent = writer->BuildEvent()) { NOlap::NTxInteractions::TTxEventContainer container(lockId, txEvent); container.AddToInteraction(InteractionsContext); - txLock.MutableEvents().emplace_back(std::move(container)); + txLock.AddTxEvent(std::move(container)); } } diff --git a/ydb/core/tx/columnshard/operations/manager.h b/ydb/core/tx/columnshard/operations/manager.h index 4937974af1c5..bb2f8d8f9eaa 100644 --- a/ydb/core/tx/columnshard/operations/manager.h +++ b/ydb/core/tx/columnshard/operations/manager.h @@ -21,9 +21,8 @@ class TLockSharingInfo: TMoveOnly { private: const ui64 LockId; const ui64 Generation; - TAtomicCounter InternalGenerationCounter = 0; std::atomic Broken = false; - TAtomicCounter WritesCounter = 0; + std::atomic Writes = false; friend class TLockFeatures; public: @@ -40,24 +39,22 @@ class TLockSharingInfo: TMoveOnly { } bool HasWrites() const { - return WritesCounter.Val(); + return Writes; } bool IsBroken() const { return Broken; } - ui64 GetCounter() const { - return InternalGenerationCounter.Val(); + ui64 GetInternalGenerationCounter() const { + return IsBroken() ? TSysTables::TLocksTable::TLock::ESetErrors::ErrorBroken : 0; } }; class TLockFeatures: TMoveOnly { private: - YDB_ACCESSOR_DEF(std::vector, WriteOperations); - YDB_ACCESSOR_DEF(std::vector, Events); - YDB_ACCESSOR(ui64, LockId, 0); - YDB_ACCESSOR(ui64, Generation, 0); + YDB_READONLY_DEF(std::vector, WriteOperations); + YDB_READONLY_DEF(std::vector, Events); std::shared_ptr SharingInfo; YDB_READONLY_DEF(THashSet, BrokeOnCommit); @@ -65,21 +62,34 @@ class TLockFeatures: TMoveOnly { YDB_READONLY_DEF(THashSet, Committed); public: + ui64 GetLockId() const { + return SharingInfo->GetLockId(); + } + + ui64 GetGeneration() const { + return SharingInfo->GetGeneration(); + } + const std::shared_ptr& GetSharingInfo() const { return SharingInfo; } ui64 GetInternalGenerationCounter() const { - return SharingInfo->GetCounter(); + return SharingInfo->GetInternalGenerationCounter(); } - void AddWrite() { - SharingInfo->WritesCounter.Inc(); + + void AddWriteOperation(const TWriteOperation::TPtr op) { + WriteOperations.push_back(op); + SharingInfo->Writes = true; + } + + void AddTxEvent(NOlap::NTxInteractions::TTxEventContainer&& container) { + Events.emplace_back(std::move(container)); } void SetBroken() { - SharingInfo->Broken = 1; - SharingInfo->InternalGenerationCounter = (i64)TSysTables::TLocksTable::TLock::ESetErrors::ErrorBroken; + SharingInfo->Broken = true; } bool IsBroken() const { @@ -109,9 +119,7 @@ class TLockFeatures: TMoveOnly { } } - TLockFeatures(const ui64 lockId, const ui64 gen) - : LockId(lockId) - , Generation(gen) { + TLockFeatures(const ui64 lockId, const ui64 gen) { SharingInfo = std::make_shared(lockId, gen); } }; @@ -199,7 +207,7 @@ class TOperationsManager { return *result; } - TWriteOperation::TPtr RegisterOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, + TWriteOperation::TPtr CreateWriteOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType); bool RegisterLock(const ui64 lockId, const ui64 generationId) { if (LockFeatures.contains(lockId)) { diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 0207b8670076..9e04423d46d5 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NColumnShard { bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { owner->Counters.GetCSCounters().WritingCounters->OnWritingTaskDequeue(TMonotonic::Now() - Created); owner->OperationsManager->RegisterLock(LockId, owner->Generation()); - auto writeOperation = owner->OperationsManager->RegisterOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType); + auto writeOperation = owner->OperationsManager->CreateWriteOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", ArrowData->GetSize())("operation_id", writeOperation->GetIdentifier())( "in_flight", owner->Counters.GetWritesMonitor()->GetWritesInFlight())( From 376e436f879bb7cc2772fdfa13190a6a1e928f04 Mon Sep 17 00:00:00 2001 From: Iurii Kravchenko Date: Mon, 23 Jun 2025 12:39:01 +0200 Subject: [PATCH 34/62] Shared metadata cache (#18544) Co-authored-by: Alexander Alexeev --- .../run/kikimr_services_initializers.cpp | 14 +++ .../run/kikimr_services_initializers.h | 7 ++ ydb/core/driver_lib/run/run.cpp | 4 + ydb/core/driver_lib/run/service_mask.h | 1 + ydb/core/protos/config.proto | 1 + ydb/core/protos/feature_flags.proto | 1 + ydb/core/testlib/basics/feature_flags.h | 1 + ydb/core/testlib/test_client.cpp | 9 ++ ydb/core/tx/columnshard/columnshard.cpp | 16 ++- .../columnshard/columnshard_private_events.h | 1 + .../data_accessor/abstract/collector.cpp | 4 +- .../data_accessor/abstract/collector.h | 42 ++++++- .../tx/columnshard/data_accessor/actor.cpp | 2 +- ydb/core/tx/columnshard/data_accessor/actor.h | 10 +- .../tx/columnshard/data_accessor/events.h | 58 ++++++--- .../data_accessor/in_mem/collector.cpp | 8 ++ .../data_accessor/in_mem/collector.h | 3 + .../data_accessor/local_db/collector.cpp | 17 ++- .../data_accessor/local_db/collector.h | 13 +- .../tx/columnshard/data_accessor/manager.h | 12 +- .../shared_metadata_accessor_cache_actor.cpp | 20 ++++ .../shared_metadata_accessor_cache_actor.h | 113 ++++++++++++++++++ ydb/core/tx/columnshard/data_accessor/ya.make | 3 +- .../test_helper/columnshard_ut_common.cpp | 11 +- .../test_helper/columnshard_ut_common.h | 2 + .../tx/schemeshard/ut_helpers/test_env.cpp | 1 + ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 6 + 27 files changed, 333 insertions(+), 47 deletions(-) create mode 100644 ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp create mode 100644 ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index efd299c1520d..c177f75ffbb0 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -143,6 +143,7 @@ #include #include #include +#include #include #include #include @@ -1115,6 +1116,18 @@ void TSharedCacheInitializer::InitializeServices( TActorSetupCmd(actor, TMailboxType::ReadAsFilled, appData->UserPoolId)); } +TSharedMetadataAccessorCacheInitializer::TSharedMetadataAccessorCacheInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) +{} + +void TSharedMetadataAccessorCacheInitializer::InitializeServices(NActors::TActorSystemSetup *setup, const TAppData *appData) { + if (appData->FeatureFlags.GetEnableSharedMetadataAccessorCache()) { + auto* actor = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(); + setup->LocalServices.emplace_back(NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(NodeId), + TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); + } +} + // TBlobCacheInitializer TBlobCacheInitializer::TBlobCacheInitializer(const TKikimrRunConfig& runConfig) @@ -2823,6 +2836,7 @@ void TKafkaProxyServiceInitializer::InitializeServices(NActors::TActorSystemSetu TActorSetupCmd(CreateDiscoveryCache(NGRpcService::KafkaEndpointId), TMailboxType::HTSwap, appData->UserPoolId) ); + setup->LocalServices.emplace_back( TActorId(), TActorSetupCmd(NKafka::CreateKafkaListener(MakePollerActorId(), settings, Config.GetKafkaProxyConfig(), diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 3d1b650577de..00b6b52057cc 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -95,6 +95,13 @@ class TSharedCacheInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; +class TSharedMetadataAccessorCacheInitializer : public IKikimrServicesInitializer { +public: +TSharedMetadataAccessorCacheInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; +}; + class TBlobCacheInitializer : public IKikimrServicesInitializer { public: TBlobCacheInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index dccfb278f36b..e0cfcd614f30 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1675,6 +1675,10 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig, ProcessMemoryInfoProvider)); + if (serviceMask.EnableSharedMetadataAccessorCache) { + sil->AddServiceInitializer(new TSharedMetadataAccessorCacheInitializer(runConfig)); + } + #if defined(ENABLE_MEMORY_TRACKING) if (serviceMask.EnableMemoryTracker) { sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig)); diff --git a/ydb/core/driver_lib/run/service_mask.h b/ydb/core/driver_lib/run/service_mask.h index 9bb31d2df8b8..4c5da67b1bc0 100644 --- a/ydb/core/driver_lib/run/service_mask.h +++ b/ydb/core/driver_lib/run/service_mask.h @@ -81,6 +81,7 @@ union TBasicKikimrServicesMask { bool EnableGroupedMemoryLimiter:1; bool EnableAwsService:1; bool EnableCompPriorities : 1; + bool EnableSharedMetadataAccessorCache : 1; }; struct { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index a57b3655b1ad..94e2ff6411bf 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1921,6 +1921,7 @@ message TColumnShardConfig { optional bool PortionMetaV0Usage = 41 [default = true]; optional uint32 PeriodicWakeupActivationPeriodMs = 42 [default = 60000]; optional bool GenerateInternalPathId = 43 [default = false]; //Be aware! IRREVERSIBLE(if set, can not be unset for a dababase) + optional uint64 SharedMetadataAccessorCacheSize = 45 [default = 1073741824]; // 1Gb } message TSchemeShardConfig { diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 147a9fcbe3f5..36f33682415f 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -203,4 +203,5 @@ message TFeatureFlags { optional bool EnableExportAutoDropping = 183 [default = false]; optional bool EnableThrottlingReport = 184 [default = false]; optional bool EnableAccessToIndexImplTables = 186 [default = false]; + optional bool EnableSharedMetadataAccessorCache = 188 [default = true]; } diff --git a/ydb/core/testlib/basics/feature_flags.h b/ydb/core/testlib/basics/feature_flags.h index 893c1ccf2cb3..bff8e3fc2a8b 100644 --- a/ydb/core/testlib/basics/feature_flags.h +++ b/ydb/core/testlib/basics/feature_flags.h @@ -77,6 +77,7 @@ class TTestFeatureFlagsHolder { FEATURE_FLAG_SETTER(EnableDatabaseAdmin) FEATURE_FLAG_SETTER(EnablePermissionsExport) FEATURE_FLAG_SETTER(EnableLocalDBBtreeIndex) + FEATURE_FLAG_SETTER(EnableSharedMetadataAccessorCache) #undef FEATURE_FLAG_SETTER }; diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 27c563af5504..883509e43306 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -76,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -1136,6 +1137,14 @@ namespace Tests { const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(NConveyorComposite::TServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } + if (Settings->FeatureFlags.GetEnableSharedMetadataAccessorCache()) { + auto* actor = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(); + + const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::HTSwap, 0); + const auto serviceId = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(Runtime->GetNodeId(nodeIdx)); + Runtime->RegisterService(serviceId, aid, nodeIdx); + } + Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0); auto sysViewService = NSysView::CreateSysViewServiceForTests(); diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index b0f60d2d6e2c..763a84a70097 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -4,6 +4,7 @@ #include "blobs_reader/actor.h" #include "counters/aggregation/table_stats.h" #include "data_accessor/actor.h" +#include "data_accessor/shared_metadata_accessor_cache_actor.h" #include "data_accessor/manager.h" #include "engines/column_engine_logs.h" #include "engines/writer/buffer/actor.h" @@ -36,7 +37,11 @@ void TColumnShard::CleanupActors(const TActorContext& ctx) { ctx.Send(ResourceSubscribeActor, new TEvents::TEvPoisonPill); ctx.Send(BufferizationInsertionWriteActorId, new TEvents::TEvPoisonPill); ctx.Send(BufferizationPortionsWriteActorId, new TEvents::TEvPoisonPill); - ctx.Send(DataAccessorsControlActorId, new TEvents::TEvPoisonPill); + if (AppData(ctx)->FeatureFlags.GetEnableSharedMetadataAccessorCache()){ + ctx.Send(DataAccessorsControlActorId, new NOlap::NDataAccessorControl::TEvClearCache(SelfId())); + } else { + ctx.Send(DataAccessorsControlActorId, new TEvents::TEvPoisonPill); + } if (!!OperationsManager) { OperationsManager->StopWriting(); } @@ -85,7 +90,7 @@ void TColumnShard::TrySwitchToWork(const TActorContext& ctx) { Counters.GetCSCounters().OnIndexMetadataLimit(NOlap::IColumnEngine::GetMetadataLimit()); EnqueueBackgroundActivities(); BackgroundSessionsManager->Start(); - ctx.Send(SelfId(), new NActors::TEvents::TEvWakeup()); + ctx.Send(SelfId(), new NActors::TEvents::TEvWakeup()); ctx.Send(SelfId(), new TEvPrivate::TEvPeriodicWakeup()); ctx.Send(SelfId(), new TEvPrivate::TEvPingSnapshotsUsage()); NYDBTest::TControllers::GetColumnShardController()->OnSwitchToWork(TabletID()); @@ -125,7 +130,12 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { ResourceSubscribeActor = ctx.Register(new NOlap::NResourceBroker::NSubscribe::TActor(TabletID(), SelfId())); BufferizationInsertionWriteActorId = ctx.Register(new NColumnShard::NWriting::TActor(TabletID(), SelfId())); BufferizationPortionsWriteActorId = ctx.Register(new NOlap::NWritingPortions::TActor(TabletID(), SelfId())); - DataAccessorsControlActorId = ctx.Register(new NOlap::NDataAccessorControl::TActor(TabletID(), SelfId())); + if (AppData(ctx)->FeatureFlags.GetEnableSharedMetadataAccessorCache()){ + DataAccessorsControlActorId = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(ctx.SelfID.NodeId()); + } else { + DataAccessorsControlActorId = ctx.Register(new NOlap::NDataAccessorControl::TActor(TabletID(), SelfId())); + } + DataAccessorsManager = std::make_shared(DataAccessorsControlActorId, SelfId()), PrioritizationClientId = NPrioritiesQueue::TCompServiceOperator::RegisterClient(); diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 52b5d8c267ef..4ea07ab0bfa8 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -68,6 +68,7 @@ struct TEvPrivate { EvAskServiceDataAccessors, EvAddPortionDataAccessor, EvRemovePortionDataAccessor, + EvClearCacheDataAccessor, EvMetadataAccessorsInfo, EvRequestFilter, diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp b/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp index 8eadb9da550b..53f3e972c42c 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp +++ b/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp @@ -15,8 +15,8 @@ TDataCategorized IGranuleDataAccessor::AnalyzeData(const TPortionsByConsumer& po return DoAnalyzeData(portions); } -void TActorAccessorsCallback::OnAccessorsFetched(std::vector&& accessors) { - NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(accessors))); +void TActorAccessorsCallback::OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) { + NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(accessors), owner)); } } // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h index 574b358c1c11..ce4ad0b42c71 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h +++ b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h @@ -1,6 +1,8 @@ #pragma once + #include #include +#include namespace NKikimr::NOlap::NDataAccessorControl { class IAccessorCallback { @@ -9,12 +11,31 @@ class IAccessorCallback { virtual ~IAccessorCallback() = default; }; -class TActorAccessorsCallback: public IAccessorCallback { +class IAccessorCallbackWithOwner { +public: + virtual void OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) = 0; + virtual ~IAccessorCallbackWithOwner() = default; +}; + +class TCallbackWrapper: public IAccessorCallback { + const std::shared_ptr Callback; + TActorId Owner; +public: + TCallbackWrapper(const std::shared_ptr& callback, const TActorId& owner) + : Callback(callback), Owner(owner) + {} + + void OnAccessorsFetched(std::vector&& accessors) override { + Callback->OnAccessorsFetched(move(accessors), Owner); + } +}; + +class TActorAccessorsCallback: public IAccessorCallbackWithOwner { private: const NActors::TActorId ActorId; public: - virtual void OnAccessorsFetched(std::vector&& accessors) override; + virtual void OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) override; TActorAccessorsCallback(const NActors::TActorId& actorId) : ActorId(actorId) { } @@ -77,14 +98,25 @@ class TDataCategorized { }; class IGranuleDataAccessor { +public: + struct TMetadataSizeProvider { + size_t operator()(const TPortionDataAccessor& data) const { + return data.GetMetadataSize(); + } + }; + + using TSharedMetadataAccessorCache = TLRUCache, TPortionDataAccessor, TNoopDelete, IGranuleDataAccessor::TMetadataSizeProvider>; private: const TInternalPathId PathId; virtual void DoAskData(THashMap&& portions, const std::shared_ptr& callback) = 0; virtual TDataCategorized DoAnalyzeData(const TPortionsByConsumer& portions) = 0; virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) = 0; + virtual void DoSetCache(std::shared_ptr) = 0; + virtual void DoSetOwner(const TActorId& owner) = 0; public: + virtual ~IGranuleDataAccessor() = default; TInternalPathId GetPathId() const { @@ -100,6 +132,12 @@ class IGranuleDataAccessor { void ModifyPortions(const std::vector& add, const std::vector& remove) { return DoModifyPortions(add, remove); } + void SetCache(std::shared_ptr cache) { + DoSetCache(cache); + } + void SetOwner(const TActorId& owner) { + DoSetOwner(owner); + } }; } // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/actor.cpp b/ydb/core/tx/columnshard/data_accessor/actor.cpp index 65680779d18e..4bad3fe4eb25 100644 --- a/ydb/core/tx/columnshard/data_accessor/actor.cpp +++ b/ydb/core/tx/columnshard/data_accessor/actor.cpp @@ -8,7 +8,7 @@ void TActor::Handle(TEvAskServiceDataAccessors::TPtr& ev) { void TActor::Bootstrap() { AccessorsCallback = std::make_shared(SelfId()); - Manager = std::make_shared(AccessorsCallback); + Manager = std::make_shared(std::make_shared(AccessorsCallback, SelfId())); Become(&TThis::StateWait); } diff --git a/ydb/core/tx/columnshard/data_accessor/actor.h b/ydb/core/tx/columnshard/data_accessor/actor.h index e21b7af85205..9eb4390c00d2 100644 --- a/ydb/core/tx/columnshard/data_accessor/actor.h +++ b/ydb/core/tx/columnshard/data_accessor/actor.h @@ -15,14 +15,14 @@ class TActor: public TActorBootstrapped { const NActors::TActorId Parent; std::shared_ptr Manager; - std::shared_ptr AccessorsCallback; + std::shared_ptr AccessorsCallback; void StartStopping() { PassAway(); } void Handle(TEvRegisterController::TPtr& ev) { - Manager->RegisterController(ev->Get()->ExtractController(), ev->Get()->IsUpdate()); + Manager->RegisterController(ev->Get()->ExtractController(), ev->Get()->GetIsUpdateFlag()); } void Handle(TEvUnregisterController::TPtr& ev) { Manager->UnregisterController(ev->Get()->GetPathId()); @@ -36,7 +36,10 @@ class TActor: public TActorBootstrapped { Manager->RemovePortion(ev->Get()->GetPortion()); } void Handle(TEvAskServiceDataAccessors::TPtr& ev); - + void Handle(TEvClearCache::TPtr&) { + Manager.reset(); + } + public: TActor(const ui64 tabletId, const TActorId& parent) : TabletId(tabletId) @@ -56,6 +59,7 @@ class TActor: public TActorBootstrapped { hFunc(TEvAskServiceDataAccessors, Handle); hFunc(TEvRemovePortion, Handle); hFunc(TEvAddPortion, Handle); + hFunc(TEvClearCache, Handle); default: AFL_VERIFY(false); } diff --git a/ydb/core/tx/columnshard/data_accessor/events.h b/ydb/core/tx/columnshard/data_accessor/events.h index 054b17e8e320..11940bbd1050 100644 --- a/ydb/core/tx/columnshard/data_accessor/events.h +++ b/ydb/core/tx/columnshard/data_accessor/events.h @@ -19,48 +19,56 @@ namespace NKikimr::NOlap::NDataAccessorControl { class TEvAddPortion: public NActors::TEventLocal { private: std::vector Accessors; + YDB_READONLY_DEF(TActorId, Owner); public: std::vector ExtractAccessors() { return std::move(Accessors); } - explicit TEvAddPortion(const TPortionDataAccessor& accessor) { - Accessors.emplace_back(accessor); + explicit TEvAddPortion(const TPortionDataAccessor& accessor, TActorId owner) + : Accessors({accessor}) + , Owner(owner) + { } - explicit TEvAddPortion(const std::vector& accessors) { - Accessors = accessors; + explicit TEvAddPortion(const std::vector& accessors, TActorId owner) + : Accessors(accessors) + , Owner(owner) + { } }; class TEvRemovePortion: public NActors::TEventLocal { private: YDB_READONLY_DEF(TPortionInfo::TConstPtr, Portion); + YDB_READONLY_DEF(TActorId, Owner); public: - explicit TEvRemovePortion(const TPortionInfo::TConstPtr& portion) - : Portion(portion) { + explicit TEvRemovePortion(const TPortionInfo::TConstPtr& portion, TActorId owner) + : Portion(portion) + , Owner(owner) + { } }; class TEvRegisterController: public NActors::TEventLocal { private: std::unique_ptr Controller; - bool IsUpdateFlag = false; + YDB_READONLY_DEF(bool, IsUpdateFlag); + YDB_READONLY_DEF(TActorId, Owner); -public: - bool IsUpdate() const { - return IsUpdateFlag; - } +public: std::unique_ptr ExtractController() { return std::move(Controller); } - explicit TEvRegisterController(std::unique_ptr&& accessor, const bool isUpdate) + explicit TEvRegisterController(std::unique_ptr&& accessor, const bool isUpdate, TActorId owner) : Controller(std::move(accessor)) - , IsUpdateFlag(isUpdate) { + , IsUpdateFlag(isUpdate) + , Owner(owner) + { } }; @@ -68,13 +76,26 @@ class TEvUnregisterController : public NActors::TEventLocal { private: YDB_READONLY_DEF(TInternalPathId, PathId); + YDB_READONLY_DEF(TActorId, Owner); public: - explicit TEvUnregisterController(const TInternalPathId pathId) - : PathId(pathId) { + explicit TEvUnregisterController(const TInternalPathId pathId, TActorId owner) + : PathId(pathId) + , Owner(owner) + { } }; +class TEvClearCache + : public NActors::TEventLocal { +private: + YDB_READONLY_DEF(TActorId, Owner); + +public: + explicit TEvClearCache(TActorId owner) : Owner(owner) {} +}; + + class TEvAskTabletDataAccessors : public NActors::TEventLocal { private: @@ -93,10 +114,13 @@ class TEvAskServiceDataAccessors : public NActors::TEventLocal { private: YDB_READONLY_DEF(std::shared_ptr, Request); + YDB_READONLY_DEF(TActorId, Owner); public: - explicit TEvAskServiceDataAccessors(const std::shared_ptr& request) - : Request(request) { + explicit TEvAskServiceDataAccessors(const std::shared_ptr& request, TActorId owner) + : Request(request) + , Owner(owner) + { } }; diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp index 137eeb5a6eb0..6916003aa375 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp @@ -28,4 +28,12 @@ void TCollector::DoModifyPortions(const std::vector& add, } } +void TCollector::DoSetCache(std::shared_ptr) { + AFL_VERIFY(false); +} + +void TCollector::DoSetOwner(const TActorId&) { + AFL_VERIFY(false); +} + } // namespace NKikimr::NOlap::NDataAccessorControl::NInMem diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h index c562d816c6f8..3646e255817a 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h @@ -11,6 +11,9 @@ class TCollector: public IGranuleDataAccessor { THashMap&& portions, const std::shared_ptr& callback) override; virtual TDataCategorized DoAnalyzeData(const TPortionsByConsumer& portions) override; virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) override; + virtual void DoSetCache(std::shared_ptr) override; + virtual void DoSetOwner(const TActorId&) override; + public: TCollector(const TInternalPathId pathId) diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp b/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp index c3ed331a9b9e..b451771f9dda 100644 --- a/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp +++ b/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp @@ -13,8 +13,9 @@ TDataCategorized TCollector::DoAnalyzeData(const TPortionsByConsumer& portions) for (auto&& c : portions.GetConsumers()) { TConsumerPortions* cPortions = nullptr; for (auto&& p : c.second.GetPortions()) { - auto it = AccessorsCache.Find(p->GetPortionId()); - if (it != AccessorsCache.End() && it.Key() == p->GetPortionId()) { + auto key = std::tuple{Owner, p->GetPathId(), p->GetPortionId()}; + if (auto it = AccessorsCache->Find(key); it != AccessorsCache->End()) { + AFL_VERIFY(it.Key() == key); result.AddFromCache(it.Value()); } else { if (!cPortions) { @@ -30,11 +31,19 @@ TDataCategorized TCollector::DoAnalyzeData(const TPortionsByConsumer& portions) void TCollector::DoModifyPortions(const std::vector& add, const std::vector& remove) { for (auto&& i : remove) { TPortionDataAccessor result = TPortionDataAccessor::BuildEmpty(); - AccessorsCache.PickOut(i, &result); + AccessorsCache->PickOut(std::tuple{Owner, GetPathId(), i}, &result); } for (auto&& i : add) { - AccessorsCache.Insert(i.GetPortionInfo().GetPortionId(), i); + AccessorsCache->Insert(std::tuple{Owner, GetPathId(), i.GetPortionInfo().GetPortionId()}, i); } } +void TCollector::DoSetCache(std::shared_ptr cache) { + AccessorsCache = cache; +} + +void TCollector::DoSetOwner(const TActorId& owner) { + Owner = TActorId(owner); +} + } // namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/collector.h b/ydb/core/tx/columnshard/data_accessor/local_db/collector.h index fc1354281984..f91d84681e94 100644 --- a/ydb/core/tx/columnshard/data_accessor/local_db/collector.h +++ b/ydb/core/tx/columnshard/data_accessor/local_db/collector.h @@ -10,24 +10,23 @@ namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB { class TCollector: public IGranuleDataAccessor { private: const NActors::TActorId TabletActorId; - struct TMetadataSizeProvider { - size_t operator()(const TPortionDataAccessor& data) { - return data.GetMetadataSize(); - } - }; + TActorId Owner{}; - TLRUCache AccessorsCache; + std::shared_ptr AccessorsCache; using TBase = IGranuleDataAccessor; virtual void DoAskData(THashMap&& portions, const std::shared_ptr& callback) override; virtual TDataCategorized DoAnalyzeData(const TPortionsByConsumer& portions) override; virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) override; + virtual void DoSetCache(std::shared_ptr) override; + virtual void DoSetOwner(const TActorId& owner) override; public: TCollector(const TInternalPathId pathId, const ui64 maxSize, const NActors::TActorId& actorId) : TBase(pathId) , TabletActorId(actorId) - , AccessorsCache(maxSize) { + { + AccessorsCache = std::make_shared(maxSize); } }; diff --git a/ydb/core/tx/columnshard/data_accessor/manager.h b/ydb/core/tx/columnshard/data_accessor/manager.h index 76a38e740bb3..781c80118b99 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/manager.h @@ -85,21 +85,21 @@ class TActorAccessorsManager: public IDataAccessorsManager { private: using TBase = IDataAccessorsManager; const NActors::TActorId ActorId; - std::shared_ptr AccessorsCallback; + std::shared_ptr AccessorsCallback; virtual void DoAskData(const std::shared_ptr& request) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(request)); + NActors::TActivationContext::Send(ActorId, std::make_unique(request, GetTabletActorId())); } virtual void DoRegisterController(std::unique_ptr&& controller, const bool update) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(controller), update)); + NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(controller), update, GetTabletActorId())); } virtual void DoUnregisterController(const TInternalPathId pathId) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(pathId)); + NActors::TActivationContext::Send(ActorId, std::make_unique(pathId, GetTabletActorId())); } virtual void DoAddPortion(const TPortionDataAccessor& accessor) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(accessor)); + NActors::TActivationContext::Send(ActorId, std::make_unique(accessor, GetTabletActorId())); } virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(portion)); + NActors::TActivationContext::Send(ActorId, std::make_unique(portion, GetTabletActorId())); } public: diff --git a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp new file mode 100644 index 000000000000..b4c4cdf5db09 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp @@ -0,0 +1,20 @@ +#include "shared_metadata_accessor_cache_actor.h" + +#include + +namespace NKikimr::NOlap::NDataAccessorControl { + +NActors::IActor* TSharedMetadataAccessorCacheActor::CreateActor() { + return new TSharedMetadataAccessorCacheActor(); +} + +void TSharedMetadataAccessorCacheActor::Bootstrap() { + AccessorsCallback = std::make_shared(SelfId()); + if (HasAppData()) { + TotalMemorySize = AppDataVerified().ColumnShardConfig.GetSharedMetadataAccessorCacheSize(); + } + MetadataCache = std::make_shared(TotalMemorySize); + Become(&TThis::StateWait); +} + +} diff --git a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h new file mode 100644 index 000000000000..da61a74cb340 --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h @@ -0,0 +1,113 @@ +#pragma once +#include "events.h" +#include "manager.h" + +#include +#include +#include + +namespace NKikimr::NOlap::NDataAccessorControl { +class TSharedMetadataAccessorCacheActor: public TActorBootstrapped { +private: + ui64 TotalMemorySize = 1 << 30; + THashMap Managers; + std::shared_ptr MetadataCache; + std::shared_ptr AccessorsCallback; + + void StartStopping() { + AFL_VERIFY(false)("shared_metadata_accessor_cache", "trying to call StartStopping()"); + PassAway(); + } + + void Handle(TEvRegisterController::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvRegisterController")("owner", ev->Get()->GetOwner()); + auto controller = ev->Get()->ExtractController(); + auto owner = ev->Get()->GetOwner(); + controller->SetCache(MetadataCache); + controller->SetOwner(owner); + auto manager = Managers.find(owner); + if (manager == Managers.end()) { + manager = Managers.emplace(owner, std::make_shared(AccessorsCallback, owner)).first; + } + manager->second.RegisterController(move(controller), ev->Get()->GetIsUpdateFlag()); + } + + void Handle(TEvUnregisterController::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvUnregisterController")("owner", ev->Get()->GetOwner()); + if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { + manager->second.UnregisterController(ev->Get()->GetPathId()); + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); + } + } + + void Handle(TEvAddPortion::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvAddPortion")("owner", ev->Get()->GetOwner()); + if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { + for (auto&& a : ev->Get()->ExtractAccessors()) { + manager->second.AddPortion(std::move(a)); + } + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); + } + } + + void Handle(TEvRemovePortion::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvRemovePortion")("owner", ev->Get()->GetOwner()); + if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { + manager->second.RemovePortion(ev->Get()->GetPortion()); + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); + } + } + + void Handle(TEvAskServiceDataAccessors::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvAskServiceDataAccessors")("owner", ev->Get()->GetOwner()); + if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { + manager->second.AskData(ev->Get()->GetRequest()); + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); + } + } + + void Handle(TEvClearCache::TPtr& ev) { + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvClearCache")("owner", ev->Get()->GetOwner()); + if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { + Managers.erase(manager); + } else { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); + } + } + +public: + + static inline TActorId MakeActorId(ui32 nodeId) { + char x[12] = {'s', 'h', 'a', 'r', 'e', + 'd', 'm', 'e', 't', 'a', 'd', 't'}; + return TActorId(nodeId, TStringBuf(x, 12)); + } + + static NActors::IActor* CreateActor(); + + TSharedMetadataAccessorCacheActor() = default; + ~TSharedMetadataAccessorCacheActor() = default; + + void Bootstrap(); + + STFUNC(StateWait) { + const NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("self_id", SelfId()); + switch (ev->GetTypeRewrite()) { + cFunc(NActors::TEvents::TEvPoison::EventType, StartStopping); + hFunc(TEvRegisterController, Handle); + hFunc(TEvUnregisterController, Handle); + hFunc(TEvAskServiceDataAccessors, Handle); + hFunc(TEvRemovePortion, Handle); + hFunc(TEvAddPortion, Handle); + hFunc(TEvClearCache, Handle); + default: + AFL_VERIFY(false); + } + } +}; + +} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/ya.make b/ydb/core/tx/columnshard/data_accessor/ya.make index f3212e91e74e..b255d527d457 100644 --- a/ydb/core/tx/columnshard/data_accessor/ya.make +++ b/ydb/core/tx/columnshard/data_accessor/ya.make @@ -3,8 +3,9 @@ LIBRARY() SRCS( actor.cpp events.cpp - request.cpp manager.cpp + request.cpp + shared_metadata_accessor_cache_actor.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index a6cd6f43f272..22f00cd0794d 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -20,6 +21,11 @@ namespace NKikimr::NTxUT { using namespace NColumnShard; using namespace Tests; +void SetupSharedMetadataAccessorCacheService(TTestActorRuntime& runtime, ui32 nodeIndex) { + runtime.AddLocalService(NKikimr::NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(runtime.GetNodeId(nodeIndex)), + TActorSetupCmd(NKikimr::NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(), TMailboxType::HTSwap, 0), nodeIndex); +} + void TTester::Setup(TTestActorRuntime& runtime) { runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); // runtime.SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_INFO); @@ -48,6 +54,9 @@ void TTester::Setup(TTestActorRuntime& runtime) { runtime.SetTxAllocatorTabletIds(ids); app.AddDomain(domain.Release()); + for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) { + SetupSharedMetadataAccessorCacheService(runtime, nodeIndex); + } SetupTabletServices(runtime, &app); runtime.UpdateCurrentTime(TInstant::Now()); @@ -545,7 +554,7 @@ namespace NKikimr::NColumnShard { Y_UNUSED(f); fields.emplace_back(idx++); } - + NTxUT::TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, snapshot); reader.SetReplyColumnIds(fields); auto rb = reader.ReadAll(); diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index 38f878b400ed..f3ad161fe45a 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -26,6 +26,8 @@ struct TIndexInfo; namespace NKikimr::NTxUT { +void SetupSharedMetadataAccessorCacheService(TTestActorRuntime& runtime, ui32 nodeIndex); + using TPlanStep = TPositiveIncreasingControlInteger; // Private events of different actors reuse the same ES_PRIVATE range diff --git a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp index 4898a8a3dafe..13def2ed7d8c 100644 --- a/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp +++ b/ydb/core/tx/schemeshard/ut_helpers/test_env.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index 77e001447e76..e09db24f1f81 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -852,6 +852,9 @@ Y_UNIT_TEST_SUITE(TOlap) { Y_UNIT_TEST(StoreStats) { TTestBasicRuntime runtime; + for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { + NKikimr::NTxUT::SetupSharedMetadataAccessorCacheService(runtime, node); + } TTestEnv env(runtime); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600)); @@ -1006,6 +1009,9 @@ Y_UNIT_TEST_SUITE(TOlap) { Y_UNIT_TEST(StoreStatsQuota) { TTestBasicRuntime runtime; + for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { + NKikimr::NTxUT::SetupSharedMetadataAccessorCacheService(runtime, node); + } TTestEnvOptions opts; opts.DisableStatsBatching(true); From 17acab102f8fc510a10e36b6b32f3e9f319e3f22 Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Tue, 24 Jun 2025 09:32:14 +0300 Subject: [PATCH 35/62] fix check MaxInternalPathId (#20071) --- ydb/core/tx/columnshard/tables_manager.cpp | 13 +++++++++---- ydb/core/tx/columnshard/tables_manager.h | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index bb158f95a601..d489d7a2c24d 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -78,7 +78,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { if (!Schema::GetSpecialValueOpt(db, Schema::EValueIds::MaxInternalPathId, maxPathId)) { return false; } - if (maxPathId) { + if (GenerateInternalPathId && maxPathId) { MaxInternalPathId =TInternalPathId::FromRawValue(*maxPathId); AFL_VERIFY(MaxInternalPathId >= GetInitialMaxInternalPathId(TabletId)); } @@ -246,8 +246,7 @@ bool TTablesManager::HasTable(const TInternalPathId pathId, const bool withDelet } TInternalPathId TTablesManager::CreateInternalPathId(const TSchemeShardLocalPathId schemeShardLocalPathId) { - if (NYDBTest::TControllers::GetColumnShardController()->IsForcedGenerateInternalPathId() || - AppData()->ColumnShardConfig.GetGenerateInternalPathId()) { + if (GenerateInternalPathId) { const auto result = TInternalPathId::FromRawValue(MaxInternalPathId.GetRawValue() + 1); MaxInternalPathId = result; return result; @@ -306,7 +305,9 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) { AFL_VERIFY(Tables.emplace(pathId, std::move(table)).second)("path_id", pathId)("size", Tables.size()); AFL_VERIFY(SchemeShardLocalToInternal.emplace(table.GetPathId().SchemeShardLocalPathId, table.GetPathId().InternalPathId).second); Schema::SaveTableSchemeShardLocalPathId(db, table.GetPathId().InternalPathId, table.GetPathId().SchemeShardLocalPathId); - Schema::SaveSpecialValue(db, Schema::EValueIds::MaxInternalPathId, MaxInternalPathId.GetRawValue()); + if (GenerateInternalPathId) { + Schema::SaveSpecialValue(db, Schema::EValueIds::MaxInternalPathId, MaxInternalPathId.GetRawValue()); + } if (PrimaryIndex) { PrimaryIndex->RegisterTable(pathId); } @@ -405,6 +406,10 @@ TTablesManager::TTablesManager(const std::shared_ptr& s , SchemaObjectsCache(schemaCache) , PortionsStats(portionsStats) , TabletId(tabletId) + , GenerateInternalPathId( + AppData()->ColumnShardConfig.GetGenerateInternalPathId() || + NYDBTest::TControllers::GetColumnShardController()->IsForcedGenerateInternalPathId() + ) , MaxInternalPathId(GetInitialMaxInternalPathId(TabletId)) { } diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 4929f0dead3f..3de1ee020ac9 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -216,6 +216,7 @@ class TTablesManager: public NOlap::IPathIdTranslator { NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; ui64 TabletId = 0; + bool GenerateInternalPathId; TInternalPathId MaxInternalPathId; friend class TTxInit; From 5105040d28998c0932e3ab885621d809ede15adb Mon Sep 17 00:00:00 2001 From: zverevgeny Date: Wed, 25 Jun 2025 03:28:30 +0300 Subject: [PATCH 36/62] Fix use after free in MoveTableProgress (#20134) --- ydb/core/tx/columnshard/tables_manager.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index d489d7a2c24d..09ce2f71e4e2 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -460,15 +460,16 @@ void TTablesManager::MoveTableProgress(NIceDb::TNiceDb& db, const TSchemeShardLo ("event", "move_table_progress")("old_path_id", oldSchemeShardLocalPathId)("new_path_id", newSchemeShardLocalPathId); AFL_VERIFY(!ResolveInternalPathId(oldSchemeShardLocalPathId)); AFL_VERIFY(!ResolveInternalPathId(newSchemeShardLocalPathId)); - const auto* internalPathId = RenamingLocalToInternal.FindPtr(oldSchemeShardLocalPathId); - AFL_VERIFY(internalPathId); - AFL_VERIFY(HasTable(*internalPathId)); - auto* table = Tables.FindPtr(*internalPathId); + const auto* pInternalPathId = RenamingLocalToInternal.FindPtr(oldSchemeShardLocalPathId); + AFL_VERIFY(pInternalPathId); + const auto internalPathId = *pInternalPathId; + AFL_VERIFY(HasTable(internalPathId)); + auto* table = Tables.FindPtr(internalPathId); AFL_VERIFY(table); table->UpdateLocalPathId(db, newSchemeShardLocalPathId); AFL_VERIFY(RenamingLocalToInternal.erase(oldSchemeShardLocalPathId)); - AFL_VERIFY(SchemeShardLocalToInternal.emplace(newSchemeShardLocalPathId, *internalPathId).second); - NYDBTest::TControllers::GetColumnShardController()->OnDeletePathId(TabletId, {*internalPathId, oldSchemeShardLocalPathId}); + AFL_VERIFY(SchemeShardLocalToInternal.emplace(newSchemeShardLocalPathId, internalPathId).second); + NYDBTest::TControllers::GetColumnShardController()->OnDeletePathId(TabletId, {internalPathId, oldSchemeShardLocalPathId}); NYDBTest::TControllers::GetColumnShardController()->OnAddPathId(TabletId, table->GetPathId()); } From d027f3cc3e09e795023958c549d3569817859f88 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Wed, 25 Jun 2025 14:35:08 +0300 Subject: [PATCH 37/62] general cache (#20004) Co-authored-by: ivanmorozov333 --- ydb/core/base/events.h | 3 + .../run/kikimr_services_initializers.cpp | 38 ++- .../run/kikimr_services_initializers.h | 13 +- ydb/core/driver_lib/run/run.cpp | 8 +- ydb/core/driver_lib/run/service_mask.h | 2 +- ydb/core/driver_lib/run/ya.make | 2 + ydb/core/protos/config.proto | 7 +- ydb/core/protos/console_config.proto | 1 + ydb/core/protos/feature_flags.proto | 2 +- ydb/core/testlib/basics/services.cpp | 9 + ydb/core/testlib/test_client.cpp | 14 +- .../blobs_action/counters/storage.h | 2 + ydb/core/tx/columnshard/columnshard.cpp | 18 +- .../columnshard/columnshard__statistics.cpp | 6 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 37 +-- ydb/core/tx/columnshard/columnshard_impl.h | 5 +- .../columnshard/columnshard_private_events.h | 14 + .../data_accessor/abstract/collector.cpp | 15 +- .../data_accessor/abstract/collector.h | 63 ++--- .../data_accessor/abstract/manager.h | 5 - .../tx/columnshard/data_accessor/actor.cpp | 15 -- ydb/core/tx/columnshard/data_accessor/actor.h | 69 ----- .../data_accessor/cache_policy/policy.cpp | 95 +++++++ .../data_accessor/cache_policy/policy.h | 76 ++++++ .../data_accessor/cache_policy/ya.make | 11 + .../tx/columnshard/data_accessor/events.cpp | 5 - .../tx/columnshard/data_accessor/events.h | 127 --------- .../data_accessor/in_mem/collector.cpp | 39 --- .../data_accessor/in_mem/collector.h | 24 -- .../data_accessor/in_mem/manager.cpp | 5 - .../data_accessor/in_mem/manager.h | 1 - .../columnshard/data_accessor/in_mem/ya.make | 1 - .../data_accessor/local_db/collector.cpp | 49 ---- .../data_accessor/local_db/collector.h | 33 --- .../data_accessor/local_db/manager.cpp | 5 - .../data_accessor/local_db/manager.h | 1 - .../data_accessor/local_db/ya.make | 1 - .../tx/columnshard/data_accessor/manager.cpp | 127 --------- .../tx/columnshard/data_accessor/manager.h | 165 +++--------- .../tx/columnshard/data_accessor/request.h | 35 ++- .../shared_metadata_accessor_cache_actor.cpp | 20 -- .../shared_metadata_accessor_cache_actor.h | 113 -------- ydb/core/tx/columnshard/data_accessor/ya.make | 6 +- .../tx/columnshard/data_reader/contexts.cpp | 13 + .../tx/columnshard/data_reader/contexts.h | 10 +- .../columnshard/data_reader/fetching_steps.h | 4 +- .../engines/changes/abstract/abstract.h | 3 +- .../engines/reader/abstract/read_context.h | 1 + .../reader/plain_reader/iterator/source.cpp | 3 +- .../reader/simple_reader/iterator/source.cpp | 2 +- .../storage/actualizer/tiering/tiering.cpp | 2 +- .../engines/storage/granule/granule.cpp | 4 +- .../engines/storage/granule/granule.h | 31 ++- .../engines/storage/granule/storage.h | 2 - .../normalizer/abstract/abstract.h | 13 + .../columnshard/normalizer/portion/chunks.cpp | 4 +- .../normalizer/portion/chunks_v0_meta.cpp | 2 +- .../normalizer/portion/leaked_blobs.cpp | 4 +- .../normalizer/portion/normalizer.cpp | 2 +- ydb/core/tx/columnshard/tables_manager.cpp | 7 +- ydb/core/tx/columnshard/tables_manager.h | 2 +- .../test_helper/columnshard_ut_common.cpp | 246 ++++++++---------- .../test_helper/columnshard_ut_common.h | 2 - ydb/core/tx/columnshard/ya.make | 1 + .../tx/conveyor_composite/service/common.cpp | 25 +- .../tx/conveyor_composite/service/common.h | 13 +- .../tx/general_cache/service/counters.cpp | 5 + ydb/core/tx/general_cache/service/counters.h | 70 +++++ ydb/core/tx/general_cache/service/manager.cpp | 5 + ydb/core/tx/general_cache/service/manager.h | 226 ++++++++++++++++ ydb/core/tx/general_cache/service/service.cpp | 5 + ydb/core/tx/general_cache/service/service.h | 64 +++++ ydb/core/tx/general_cache/service/ya.make | 13 + ydb/core/tx/general_cache/source/abstract.cpp | 5 + ydb/core/tx/general_cache/source/abstract.h | 35 +++ ydb/core/tx/general_cache/source/events.cpp | 5 + ydb/core/tx/general_cache/source/events.h | 91 +++++++ ydb/core/tx/general_cache/source/ya.make | 13 + ydb/core/tx/general_cache/usage/abstract.cpp | 5 + ydb/core/tx/general_cache/usage/abstract.h | 25 ++ ydb/core/tx/general_cache/usage/config.cpp | 35 +++ ydb/core/tx/general_cache/usage/config.h | 28 ++ ydb/core/tx/general_cache/usage/events.cpp | 5 + ydb/core/tx/general_cache/usage/events.h | 56 ++++ ydb/core/tx/general_cache/usage/service.cpp | 5 + ydb/core/tx/general_cache/usage/service.h | 45 ++++ ydb/core/tx/general_cache/usage/ya.make | 16 ++ ydb/core/tx/general_cache/ya.make | 12 + ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp | 6 - ydb/core/tx/ya.make | 1 + ydb/library/services/services.proto | 2 + 91 files changed, 1355 insertions(+), 1106 deletions(-) delete mode 100644 ydb/core/tx/columnshard/data_accessor/actor.cpp delete mode 100644 ydb/core/tx/columnshard/data_accessor/actor.h create mode 100644 ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp create mode 100644 ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h create mode 100644 ydb/core/tx/columnshard/data_accessor/cache_policy/ya.make delete mode 100644 ydb/core/tx/columnshard/data_accessor/events.cpp delete mode 100644 ydb/core/tx/columnshard/data_accessor/events.h delete mode 100644 ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp delete mode 100644 ydb/core/tx/columnshard/data_accessor/in_mem/collector.h delete mode 100644 ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp delete mode 100644 ydb/core/tx/columnshard/data_accessor/local_db/collector.h delete mode 100644 ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp delete mode 100644 ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h create mode 100644 ydb/core/tx/general_cache/service/counters.cpp create mode 100644 ydb/core/tx/general_cache/service/counters.h create mode 100644 ydb/core/tx/general_cache/service/manager.cpp create mode 100644 ydb/core/tx/general_cache/service/manager.h create mode 100644 ydb/core/tx/general_cache/service/service.cpp create mode 100644 ydb/core/tx/general_cache/service/service.h create mode 100644 ydb/core/tx/general_cache/service/ya.make create mode 100644 ydb/core/tx/general_cache/source/abstract.cpp create mode 100644 ydb/core/tx/general_cache/source/abstract.h create mode 100644 ydb/core/tx/general_cache/source/events.cpp create mode 100644 ydb/core/tx/general_cache/source/events.h create mode 100644 ydb/core/tx/general_cache/source/ya.make create mode 100644 ydb/core/tx/general_cache/usage/abstract.cpp create mode 100644 ydb/core/tx/general_cache/usage/abstract.h create mode 100644 ydb/core/tx/general_cache/usage/config.cpp create mode 100644 ydb/core/tx/general_cache/usage/config.h create mode 100644 ydb/core/tx/general_cache/usage/events.cpp create mode 100644 ydb/core/tx/general_cache/usage/events.h create mode 100644 ydb/core/tx/general_cache/usage/service.cpp create mode 100644 ydb/core/tx/general_cache/usage/service.h create mode 100644 ydb/core/tx/general_cache/usage/ya.make create mode 100644 ydb/core/tx/general_cache/ya.make diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index eb9010f205d3..f258c3ab2b54 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -186,6 +186,9 @@ struct TKikimrEvents : TEvents { ES_PRIORITY_QUEUE = 4263, ES_SOLOMON_PROVIDER = 4264, ES_CONVEYOR_COMPOSITE = 4265, + ES_GENERAL_CACHE_PUBLIC = 4266, + ES_GENERAL_CACHE_SOURCE = 4267, + }; }; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index c177f75ffbb0..3722a1a7771d 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -143,7 +143,6 @@ #include #include #include -#include #include #include #include @@ -190,6 +189,8 @@ #include #include #include +#include +#include #include #include @@ -1116,18 +1117,6 @@ void TSharedCacheInitializer::InitializeServices( TActorSetupCmd(actor, TMailboxType::ReadAsFilled, appData->UserPoolId)); } -TSharedMetadataAccessorCacheInitializer::TSharedMetadataAccessorCacheInitializer(const TKikimrRunConfig& runConfig) - : IKikimrServicesInitializer(runConfig) -{} - -void TSharedMetadataAccessorCacheInitializer::InitializeServices(NActors::TActorSystemSetup *setup, const TAppData *appData) { - if (appData->FeatureFlags.GetEnableSharedMetadataAccessorCache()) { - auto* actor = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(); - setup->LocalServices.emplace_back(NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(NodeId), - TActorSetupCmd(actor, TMailboxType::HTSwap, appData->UserPoolId)); - } -} - // TBlobCacheInitializer TBlobCacheInitializer::TBlobCacheInitializer(const TKikimrRunConfig& runConfig) @@ -2250,6 +2239,29 @@ void TCompPrioritiesInitializer::InitializeServices(NActors::TActorSystemSetup* } } +TGeneralCachePortionsMetadataInitializer::TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig) + : IKikimrServicesInitializer(runConfig) { +} + +void TGeneralCachePortionsMetadataInitializer::InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) { + auto serviceConfig = NGeneralCache::NPublic::TConfig::BuildFromProto(Config.GetPortionsMetadataCache()); + if (serviceConfig.IsFail()) { + AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse portions metadata cache config")("action", "default_usage")( + "error", serviceConfig.GetErrorMessage())("default", NGeneralCache::NPublic::TConfig::BuildDefault().DebugString()); + serviceConfig = NGeneralCache::NPublic::TConfig::BuildDefault(); + } + AFL_VERIFY(!serviceConfig.IsFail()); + + TIntrusivePtr<::NMonitoring::TDynamicCounters> tabletGroup = GetServiceCounters(appData->Counters, "tablets"); + TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorGroup = tabletGroup->GetSubgroup("type", "TX_GENERAL_CACHE_PORTIONS_METADATA"); + + auto service = NGeneralCache::TServiceOperator::CreateService(*serviceConfig, conveyorGroup); + + setup->LocalServices.push_back( + std::make_pair(NGeneralCache::TServiceOperator::MakeServiceId(NodeId), + TActorSetupCmd(service, TMailboxType::HTSwap, appData->UserPoolId))); +} + TCompositeConveyorInitializer::TCompositeConveyorInitializer(const TKikimrRunConfig& runConfig) : IKikimrServicesInitializer(runConfig) { } diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index 00b6b52057cc..4ea093d87e75 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -95,13 +95,6 @@ class TSharedCacheInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; }; -class TSharedMetadataAccessorCacheInitializer : public IKikimrServicesInitializer { -public: -TSharedMetadataAccessorCacheInitializer(const TKikimrRunConfig& runConfig); - - void InitializeServices(NActors::TActorSystemSetup *setup, const NKikimr::TAppData *appData) override; -}; - class TBlobCacheInitializer : public IKikimrServicesInitializer { public: TBlobCacheInitializer(const TKikimrRunConfig& runConfig); @@ -435,6 +428,12 @@ class TCompositeConveyorInitializer : public IKikimrServicesInitializer { void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TGeneralCachePortionsMetadataInitializer: public IKikimrServicesInitializer { +public: + TGeneralCachePortionsMetadataInitializer(const TKikimrRunConfig& runConfig); + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TExternalIndexInitializer: public IKikimrServicesInitializer { public: TExternalIndexInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index e0cfcd614f30..a9343b11aebf 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1675,10 +1675,6 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TMemProfMonitorInitializer(runConfig, ProcessMemoryInfoProvider)); - if (serviceMask.EnableSharedMetadataAccessorCache) { - sil->AddServiceInitializer(new TSharedMetadataAccessorCacheInitializer(runConfig)); - } - #if defined(ENABLE_MEMORY_TRACKING) if (serviceMask.EnableMemoryTracker) { sil->AddServiceInitializer(new TMemoryTrackerInitializer(runConfig)); @@ -1716,6 +1712,10 @@ TIntrusivePtr TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TCompositeConveyorInitializer(runConfig)); } + if (serviceMask.EnableGeneralCachePortionsMetadata) { + sil->AddServiceInitializer(new TGeneralCachePortionsMetadataInitializer(runConfig)); + } + if (serviceMask.EnableCms) { sil->AddServiceInitializer(new TCmsServiceInitializer(runConfig)); } diff --git a/ydb/core/driver_lib/run/service_mask.h b/ydb/core/driver_lib/run/service_mask.h index 4c5da67b1bc0..0487ece46062 100644 --- a/ydb/core/driver_lib/run/service_mask.h +++ b/ydb/core/driver_lib/run/service_mask.h @@ -81,7 +81,7 @@ union TBasicKikimrServicesMask { bool EnableGroupedMemoryLimiter:1; bool EnableAwsService:1; bool EnableCompPriorities : 1; - bool EnableSharedMetadataAccessorCache : 1; + bool EnableGeneralCachePortionsMetadata: 1; }; struct { diff --git a/ydb/core/driver_lib/run/ya.make b/ydb/core/driver_lib/run/ya.make index b5cf427c036e..0b2813a18763 100644 --- a/ydb/core/driver_lib/run/ya.make +++ b/ydb/core/driver_lib/run/ya.make @@ -106,6 +106,8 @@ PEERDIR( ydb/core/tx ydb/core/tx/columnshard ydb/core/tx/conveyor/service + ydb/core/tx/general_cache + ydb/core/tx/columnshard/data_accessor/cache_policy ydb/core/tx/coordinator ydb/core/tx/datashard ydb/core/tx/limiter/grouped_memory/usage diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 94e2ff6411bf..119920a067c9 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1921,7 +1921,6 @@ message TColumnShardConfig { optional bool PortionMetaV0Usage = 41 [default = true]; optional uint32 PeriodicWakeupActivationPeriodMs = 42 [default = 60000]; optional bool GenerateInternalPathId = 43 [default = false]; //Be aware! IRREVERSIBLE(if set, can not be unset for a dababase) - optional uint64 SharedMetadataAccessorCacheSize = 45 [default = 1073741824]; // 1Gb } message TSchemeShardConfig { @@ -2262,6 +2261,11 @@ message TLabel { optional string Value = 2; } +message TGeneralCacheConfig { + optional uint64 MemoryLimit = 1; + optional uint64 DirectInflightLimit = 2; +} + message TAppConfig { option (NMarkers.Root) = true; optional TActorSystemConfig ActorSystemConfig = 1; @@ -2360,6 +2364,7 @@ message TAppConfig { optional string StartupConfigYaml = 107; optional string StartupStorageYaml = 108; optional TCompositeConveyorConfig CompositeConveyorConfig = 109; + optional TGeneralCacheConfig PortionsMetadataCache = 110; } message TYdbVersion { diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 2943e7e33234..7357218c23a5 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -150,6 +150,7 @@ message TConfigItem { ClusterYamlConfigItem = 101; CompositeConveyorConfigItem = 109; + PortionsMetadataCacheItem = 110; // synthetic kinds for audit purposes only DatabaseYamlConfigChangeItem = 32767; diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 36f33682415f..f00f7b27403c 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -203,5 +203,5 @@ message TFeatureFlags { optional bool EnableExportAutoDropping = 183 [default = false]; optional bool EnableThrottlingReport = 184 [default = false]; optional bool EnableAccessToIndexImplTables = 186 [default = false]; - optional bool EnableSharedMetadataAccessorCache = 188 [default = true]; + optional bool EnableSharedMetadataAccessorCache = 188 [default = true, deprecated = true]; } diff --git a/ydb/core/testlib/basics/services.cpp b/ydb/core/testlib/basics/services.cpp index b7920473479f..6e37bac8a041 100644 --- a/ydb/core/testlib/basics/services.cpp +++ b/ydb/core/testlib/basics/services.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -166,6 +167,13 @@ namespace NPDisk { nodeIndex); } + void SetupCSMetadataCache(TTestActorRuntime& runtime, ui32 nodeIndex) { + auto* actor = NOlap::NDataAccessorControl::TGeneralCache::CreateService( + NGeneralCache::NPublic::TConfig::BuildDefault(), runtime.GetDynamicCounters(nodeIndex)); + runtime.AddLocalService(NOlap::NDataAccessorControl::TGeneralCache::MakeServiceId(runtime.GetNodeId(nodeIndex)), + TActorSetupCmd(actor, TMailboxType::ReadAsFilled, 0), nodeIndex); + } + void SetupBlobCache(TTestActorRuntime& runtime, ui32 nodeIndex) { runtime.AddLocalService(NBlobCache::MakeBlobCacheServiceId(), @@ -376,6 +384,7 @@ namespace NPDisk { SetupResourceBroker(runtime, nodeIndex, app.ResourceBrokerConfig); SetupSharedPageCache(runtime, nodeIndex, sharedCacheConfig ? *sharedCacheConfig : defaultSharedCacheConfig); SetupBlobCache(runtime, nodeIndex); + SetupCSMetadataCache(runtime, nodeIndex); SetupSysViewService(runtime, nodeIndex); SetupQuoterService(runtime, nodeIndex); SetupStatService(runtime, nodeIndex); diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index 883509e43306..c25be1a6d3f3 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -76,7 +76,6 @@ #include #include #include -#include #include #include #include @@ -130,6 +129,8 @@ #include #include #include +#include +#include #include #include @@ -1137,14 +1138,11 @@ namespace Tests { const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); Runtime->RegisterService(NConveyorComposite::TServiceOperator::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } - if (Settings->FeatureFlags.GetEnableSharedMetadataAccessorCache()) { - auto* actor = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(); - - const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::HTSwap, 0); - const auto serviceId = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(Runtime->GetNodeId(nodeIdx)); - Runtime->RegisterService(serviceId, aid, nodeIdx); + { + auto* actor = NOlap::NDataAccessorControl::TGeneralCache::CreateService(NGeneralCache::NPublic::TConfig::BuildDefault(), new ::NMonitoring::TDynamicCounters()); + const auto aid = Runtime->Register(actor, nodeIdx, appData.UserPoolId, TMailboxType::Revolving, 0); + Runtime->RegisterService(NOlap::NDataAccessorControl::TGeneralCache::MakeServiceId(Runtime->GetNodeId(nodeIdx)), aid, nodeIdx); } - Runtime->Register(CreateLabelsMaintainer({}), nodeIdx, appData.SystemPoolId, TMailboxType::Revolving, 0); auto sysViewService = NSysView::CreateSysViewServiceForTests(); diff --git a/ydb/core/tx/columnshard/blobs_action/counters/storage.h b/ydb/core/tx/columnshard/blobs_action/counters/storage.h index 9a94ccfc4a85..4ecf83fbc0b9 100644 --- a/ydb/core/tx/columnshard/blobs_action/counters/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/counters/storage.h @@ -30,6 +30,8 @@ enum class EConsumer { WRITING_OPERATOR, NORMALIZER, STATISTICS, + STATISTICS_FLUSH, + FETCH_ON_LOAD, UNDEFINED, COUNT diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 763a84a70097..9f2b694532c7 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -3,9 +3,6 @@ #include "bg_tasks/manager/manager.h" #include "blobs_reader/actor.h" #include "counters/aggregation/table_stats.h" -#include "data_accessor/actor.h" -#include "data_accessor/shared_metadata_accessor_cache_actor.h" -#include "data_accessor/manager.h" #include "engines/column_engine_logs.h" #include "engines/writer/buffer/actor.h" #include "engines/writer/buffer/actor2.h" @@ -37,11 +34,6 @@ void TColumnShard::CleanupActors(const TActorContext& ctx) { ctx.Send(ResourceSubscribeActor, new TEvents::TEvPoisonPill); ctx.Send(BufferizationInsertionWriteActorId, new TEvents::TEvPoisonPill); ctx.Send(BufferizationPortionsWriteActorId, new TEvents::TEvPoisonPill); - if (AppData(ctx)->FeatureFlags.GetEnableSharedMetadataAccessorCache()){ - ctx.Send(DataAccessorsControlActorId, new NOlap::NDataAccessorControl::TEvClearCache(SelfId())); - } else { - ctx.Send(DataAccessorsControlActorId, new TEvents::TEvPoisonPill); - } if (!!OperationsManager) { OperationsManager->StopWriting(); } @@ -130,14 +122,8 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { ResourceSubscribeActor = ctx.Register(new NOlap::NResourceBroker::NSubscribe::TActor(TabletID(), SelfId())); BufferizationInsertionWriteActorId = ctx.Register(new NColumnShard::NWriting::TActor(TabletID(), SelfId())); BufferizationPortionsWriteActorId = ctx.Register(new NOlap::NWritingPortions::TActor(TabletID(), SelfId())); - if (AppData(ctx)->FeatureFlags.GetEnableSharedMetadataAccessorCache()){ - DataAccessorsControlActorId = NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(ctx.SelfID.NodeId()); - } else { - DataAccessorsControlActorId = ctx.Register(new NOlap::NDataAccessorControl::TActor(TabletID(), SelfId())); - } - - DataAccessorsManager = std::make_shared(DataAccessorsControlActorId, SelfId()), - + DataAccessorsManager = std::make_shared(SelfId()); + NormalizerController.SetDataAccessorsManager(DataAccessorsManager); PrioritizationClientId = NPrioritiesQueue::TCompServiceOperator::RegisterClient(); Execute(CreateTxInitSchema(), ctx); } diff --git a/ydb/core/tx/columnshard/columnshard__statistics.cpp b/ydb/core/tx/columnshard/columnshard__statistics.cpp index de1096342a1d..b755f5b43f59 100644 --- a/ydb/core/tx/columnshard/columnshard__statistics.cpp +++ b/ydb/core/tx/columnshard/columnshard__statistics.cpp @@ -1,6 +1,7 @@ #include "columnshard.h" #include "columnshard_impl.h" +#include "data_accessor/cache_policy/policy.h" #include "ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/meta.h" #include @@ -253,7 +254,8 @@ class TColumnPortionsAccumulator { return; } Result->AddWaitingTask(); - std::shared_ptr request = std::make_shared("STATISTICS_FLUSH"); + std::shared_ptr request = + std::make_shared(NOlap::NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::STATISTICS_FLUSH); for (auto&& i : Portions) { request->AddPortion(i); } @@ -304,7 +306,7 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev, columnTagsRequested = std::set(allColumnIds.begin(), allColumnIds.end()); } - NOlap::TDataAccessorsRequest request("STATISTICS"); + NOlap::TDataAccessorsRequest request(NOlap::NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::STATISTICS); std::shared_ptr resultAccumulator = std::make_shared(columnTagsRequested, ev->Sender, ev->Cookie, std::move(response)); auto versionedIndex = std::make_shared(index.GetVersionedIndex()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 2342fb69fb8d..4659caf6385e 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -85,8 +85,7 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod()) , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval()) , InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters()) - , TablesManager(StoragesManager, std::make_shared(nullptr), nullptr, - Counters.GetPortionIndexCounters(), info->TabletID) + , TablesManager(StoragesManager, nullptr, nullptr, Counters.GetPortionIndexCounters(), info->TabletID) , Subscribers(std::make_shared(*this)) , PipeClientCache(NTabletPipe::CreateBoundedClientCache(new NTabletPipe::TBoundedClientCacheConfig(), GetPipeClientConfig())) , CompactTaskSubscription(NOlap::TCompactColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) @@ -1061,13 +1060,14 @@ void TColumnShard::Handle(NOlap::NDataSharing::NEvents::TEvFinishedFromSource::T class TPortionConstructorV2 { private: - NOlap::TPortionInfo::TConstPtr PortionInfo; + YDB_READONLY_DEF(NOlap::TPortionInfo::TConstPtr, PortionInfo); std::optional Records; std::optional> Indexes; public: TPortionConstructorV2(const NOlap::TPortionInfo::TConstPtr& portionInfo) : PortionInfo(portionInfo) { + AFL_VERIFY(PortionInfo); } bool IsReady() const { @@ -1155,35 +1155,38 @@ class TTxAskPortionChunks: public TTransactionBase { bool reask = false; NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("event", "TTxAskPortionChunks::Execute"); for (auto&& i : PortionsByPath) { + const auto& granule = Self->GetIndexAs().GetGranuleVerified(i.first); for (auto&& c : i.second.GetConsumers()) { NActors::TLogContextGuard lcGuard = NActors::TLogContextBuilder::Build()("consumer", c.first)("path_id", i.first); - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("size", c.second.GetPortions().size()); - for (auto&& p : c.second.GetPortions()) { - auto itPortionConstructor = Constructors.find(p->GetAddress()); + AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("size", c.second.GetPortionsCount()); + for (auto&& portion : c.second.GetPortions(granule)) { + const ui64 p = portion->GetPortionId(); + const NOlap::TPortionAddress pAddress = portion->GetAddress(); + auto itPortionConstructor = Constructors.find(pAddress); if (itPortionConstructor == Constructors.end()) { - TPortionConstructorV2 constructor(p); - itPortionConstructor = Constructors.emplace(p->GetAddress(), std::move(constructor)).first; + TPortionConstructorV2 constructor(portion); + itPortionConstructor = Constructors.emplace(pAddress, std::move(constructor)).first; } else if (itPortionConstructor->second.IsReady()) { continue; } if (!itPortionConstructor->second.HasRecords()) { - auto rowset = - db.Table().Key(p->GetPathId().GetRawValue(), p->GetPortionId()).Select(); + auto rowset = db.Table().Key(i.first.GetRawValue(), p).Select(); if (!rowset.IsReady()) { reask = true; } else { - AFL_VERIFY(!rowset.EndOfSet())("path_id", p->GetPathId())("portion_id", p->GetPortionId())( - "debug", p->DebugString(true)); + AFL_VERIFY(!rowset.EndOfSet())("path_id", i.first)("portion_id", p)( + "debug", itPortionConstructor->second.GetPortionInfo()->DebugString(true)); NOlap::TColumnChunkLoadContextV2 info(rowset); itPortionConstructor->second.SetRecords(std::move(info)); } } if (!itPortionConstructor->second.HasIndexes()) { - if (!p->GetSchema(Self->GetIndexAs().GetVersionedIndex())->GetIndexesCount()) { + if (!itPortionConstructor->second.GetPortionInfo() + ->GetSchema(Self->GetIndexAs().GetVersionedIndex()) + ->GetIndexesCount()) { itPortionConstructor->second.SetIndexes({}); } else { - auto rowset = - db.Table().Prefix(p->GetPathId().GetRawValue(), p->GetPortionId()).Select(); + auto rowset = db.Table().Prefix(i.first.GetRawValue(), p).Select(); if (!rowset.IsReady()) { reask = true; } else { @@ -1223,7 +1226,7 @@ class TTxAskPortionChunks: public TTransactionBase { } }; -void TColumnShard::Handle(NOlap::NDataAccessorControl::TEvAskTabletDataAccessors::TPtr& ev, const TActorContext& /*ctx*/) { +void TColumnShard::Handle(NColumnShard::TEvPrivate::TEvAskTabletDataAccessors::TPtr& ev, const TActorContext& /*ctx*/) { Execute(new TTxAskPortionChunks(this, ev->Get()->GetCallback(), std::move(ev->Get()->DetachPortions()))); } @@ -1316,7 +1319,7 @@ void TColumnShard::Enqueue(STFUNC_SIG) { switch (ev->GetTypeRewrite()) { HFunc(TEvPrivate::TEvTieringModified, HandleInit); HFunc(TEvPrivate::TEvNormalizerResult, Handle); - HFunc(NOlap::NDataAccessorControl::TEvAskTabletDataAccessors, Handle); + HFunc(TEvPrivate::TEvAskTabletDataAccessors, Handle); HFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); default: AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "unexpected event in enqueue"); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index f981afb7d3c7..1f510242fa97 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -292,7 +292,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa void Handle(NOlap::NDataSharing::NEvents::TEvFinishedFromSource::TPtr& ev, const TActorContext& ctx); void Handle(NOlap::NDataSharing::NEvents::TEvAckFinishToSource::TPtr& ev, const TActorContext& ctx); void Handle(NOlap::NDataSharing::NEvents::TEvAckFinishFromInitiator::TPtr& ev, const TActorContext& ctx); - void Handle(NOlap::NDataAccessorControl::TEvAskTabletDataAccessors::TPtr& ev, const TActorContext& ctx); + void Handle(NColumnShard::TEvPrivate::TEvAskTabletDataAccessors::TPtr& ev, const TActorContext& ctx); void Handle(TEvTxProxySchemeCache::TEvWatchNotifyUpdated::TPtr& ev, const TActorContext& ctx); void HandleInit(TEvPrivate::TEvTieringModified::TPtr& ev, const TActorContext&); @@ -450,7 +450,7 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa HFunc(NOlap::NDataSharing::NEvents::TEvFinishedFromSource, Handle); HFunc(NOlap::NDataSharing::NEvents::TEvAckFinishToSource, Handle); HFunc(NOlap::NDataSharing::NEvents::TEvAckFinishFromInitiator, Handle); - HFunc(NOlap::NDataAccessorControl::TEvAskTabletDataAccessors, Handle); + HFunc(NColumnShard::TEvPrivate::TEvAskTabletDataAccessors, Handle); HFunc(TEvTxProxySchemeCache::TEvWatchNotifyUpdated, Handle); default: @@ -506,7 +506,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa TActorId ResourceSubscribeActor; TActorId BufferizationInsertionWriteActorId; TActorId BufferizationPortionsWriteActorId; - TActorId DataAccessorsControlActorId; NOlap::NDataAccessorControl::TDataAccessorsManagerContainer DataAccessorsManager; TActorId StatsReportPipe; diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 4ea07ab0bfa8..21db6025e4df 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -107,6 +107,20 @@ struct TEvPrivate { } }; + class TEvAskTabletDataAccessors + : public NActors::TEventLocal { + private: + using TPortions = THashMap; + YDB_ACCESSOR_DEF(TPortions, Portions); + YDB_READONLY_DEF(std::shared_ptr, Callback); + + public: + explicit TEvAskTabletDataAccessors(TPortions&& portions, const std::shared_ptr& callback) + : Portions(std::move(portions)) + , Callback(callback) { + } + }; + class TEvStartCompaction: public NActors::TEventLocal { private: YDB_READONLY_DEF(std::shared_ptr, Guard); diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp b/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp index 53f3e972c42c..754c41e2db31 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp +++ b/ydb/core/tx/columnshard/data_accessor/abstract/collector.cpp @@ -1,7 +1,7 @@ #include "collector.h" -#include #include +#include namespace NKikimr::NOlap::NDataAccessorControl { @@ -15,8 +15,17 @@ TDataCategorized IGranuleDataAccessor::AnalyzeData(const TPortionsByConsumer& po return DoAnalyzeData(portions); } -void TActorAccessorsCallback::OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) { - NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(accessors), owner)); +std::vector TConsumerPortions::GetPortions(const TGranuleMeta& granule) const { + std::vector result; + result.reserve(PortionIds.size()); + for (auto&& i : PortionIds) { + result.emplace_back(granule.GetPortionVerifiedPtr(i, false)); + } + return result; +} + +void TConsumerPortions::AddPortion(const std::shared_ptr& p) { + PortionIds.emplace_back(p->GetPortionId()); } } // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h index ce4ad0b42c71..48d6fd59f205 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h +++ b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h @@ -1,9 +1,14 @@ #pragma once - +#include #include #include #include +namespace NKikimr::NOlap { +class TGranuleMeta; +class TPortionInfo; +} + namespace NKikimr::NOlap::NDataAccessorControl { class IAccessorCallback { public: @@ -11,60 +16,32 @@ class IAccessorCallback { virtual ~IAccessorCallback() = default; }; -class IAccessorCallbackWithOwner { -public: - virtual void OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) = 0; - virtual ~IAccessorCallbackWithOwner() = default; -}; - -class TCallbackWrapper: public IAccessorCallback { - const std::shared_ptr Callback; - TActorId Owner; -public: - TCallbackWrapper(const std::shared_ptr& callback, const TActorId& owner) - : Callback(callback), Owner(owner) - {} - - void OnAccessorsFetched(std::vector&& accessors) override { - Callback->OnAccessorsFetched(move(accessors), Owner); - } -}; - -class TActorAccessorsCallback: public IAccessorCallbackWithOwner { -private: - const NActors::TActorId ActorId; - -public: - virtual void OnAccessorsFetched(std::vector&& accessors, const TActorId& owner) override; - TActorAccessorsCallback(const NActors::TActorId& actorId) - : ActorId(actorId) { - } -}; - class TConsumerPortions { private: - YDB_READONLY_DEF(TString, ConsumerId); - YDB_READONLY_DEF(std::vector, Portions); + YDB_READONLY_DEF(std::vector, PortionIds); public: - void AddPortion(const TPortionInfo::TConstPtr& p) { - Portions.emplace_back(p); + void AddPortion(const std::shared_ptr& p); + void AddPortion(const ui64 portionId) { + PortionIds.emplace_back(portionId); } - TConsumerPortions(const TString& consumerId) - : ConsumerId(consumerId) { + ui32 GetPortionsCount() const { + return PortionIds.size(); } + + std::vector GetPortions(const TGranuleMeta& granule) const; }; class TPortionsByConsumer { private: - THashMap Consumers; + THashMap Consumers; public: ui64 GetPortionsCount() const { ui64 result = 0; for (auto&& i : Consumers) { - result += i.second.GetPortions().size(); + result += i.second.GetPortionsCount(); } return result; } @@ -73,15 +50,15 @@ class TPortionsByConsumer { return Consumers.empty(); } - TConsumerPortions& UpsertConsumer(const TString& consumerId) { - auto it = Consumers.find(consumerId); + TConsumerPortions& UpsertConsumer(const NGeneralCache::TPortionsMetadataCachePolicy::EConsumer consumer) { + auto it = Consumers.find(consumer); if (it == Consumers.end()) { - it = Consumers.emplace(consumerId, consumerId).first; + it = Consumers.emplace(consumer, TConsumerPortions()).first; } return it->second; } - const THashMap& GetConsumers() const { + const THashMap& GetConsumers() const { return Consumers; } }; diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/manager.h b/ydb/core/tx/columnshard/data_accessor/abstract/manager.h index ec1516c1cdbf..e2a26670671a 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/abstract/manager.h @@ -12,7 +12,6 @@ class TGranuleMeta; namespace NKikimr::NOlap::NDataAccessorControl { class IMetadataMemoryManager { private: - virtual std::unique_ptr DoBuildCollector(const TInternalPathId pathId) = 0; virtual std::shared_ptr DoBuildLoader( const TVersionedIndex& versionedIndex, TGranuleMeta* granule, const std::shared_ptr& dsGroupSelector) = 0; @@ -22,10 +21,6 @@ class IMetadataMemoryManager { return false; } - std::unique_ptr BuildCollector(const TInternalPathId pathId) { - return DoBuildCollector(pathId); - } - std::shared_ptr BuildLoader( const TVersionedIndex& versionedIndex, TGranuleMeta* granule, const std::shared_ptr& dsGroupSelector) { return DoBuildLoader(versionedIndex, granule, dsGroupSelector); diff --git a/ydb/core/tx/columnshard/data_accessor/actor.cpp b/ydb/core/tx/columnshard/data_accessor/actor.cpp deleted file mode 100644 index 4bad3fe4eb25..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/actor.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "actor.h" - -namespace NKikimr::NOlap::NDataAccessorControl { - -void TActor::Handle(TEvAskServiceDataAccessors::TPtr& ev) { - Manager->AskData(ev->Get()->GetRequest()); -} - -void TActor::Bootstrap() { - AccessorsCallback = std::make_shared(SelfId()); - Manager = std::make_shared(std::make_shared(AccessorsCallback, SelfId())); - Become(&TThis::StateWait); -} - -} diff --git a/ydb/core/tx/columnshard/data_accessor/actor.h b/ydb/core/tx/columnshard/data_accessor/actor.h deleted file mode 100644 index 9eb4390c00d2..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/actor.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once -#include "events.h" -#include "manager.h" - -#include "abstract/collector.h" - -#include -#include - -namespace NKikimr::NOlap::NDataAccessorControl { - -class TActor: public TActorBootstrapped { -private: - const ui64 TabletId; - const NActors::TActorId Parent; - std::shared_ptr Manager; - - std::shared_ptr AccessorsCallback; - - void StartStopping() { - PassAway(); - } - - void Handle(TEvRegisterController::TPtr& ev) { - Manager->RegisterController(ev->Get()->ExtractController(), ev->Get()->GetIsUpdateFlag()); - } - void Handle(TEvUnregisterController::TPtr& ev) { - Manager->UnregisterController(ev->Get()->GetPathId()); - } - void Handle(TEvAddPortion::TPtr& ev) { - for (auto&& a : ev->Get()->ExtractAccessors()) { - Manager->AddPortion(std::move(a)); - } - } - void Handle(TEvRemovePortion::TPtr& ev) { - Manager->RemovePortion(ev->Get()->GetPortion()); - } - void Handle(TEvAskServiceDataAccessors::TPtr& ev); - void Handle(TEvClearCache::TPtr&) { - Manager.reset(); - } - -public: - TActor(const ui64 tabletId, const TActorId& parent) - : TabletId(tabletId) - , Parent(parent) { - Y_UNUSED(TabletId); - } - ~TActor() = default; - - void Bootstrap(); - - STFUNC(StateWait) { - const NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("self_id", SelfId())("tablet_id", TabletId)("parent", Parent); - switch (ev->GetTypeRewrite()) { - cFunc(NActors::TEvents::TEvPoison::EventType, StartStopping); - hFunc(TEvRegisterController, Handle); - hFunc(TEvUnregisterController, Handle); - hFunc(TEvAskServiceDataAccessors, Handle); - hFunc(TEvRemovePortion, Handle); - hFunc(TEvAddPortion, Handle); - hFunc(TEvClearCache, Handle); - default: - AFL_VERIFY(false); - } - } -}; - -} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp new file mode 100644 index 000000000000..55134427e8ca --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp @@ -0,0 +1,95 @@ +#include "policy.h" + +#include +#include +#include +#include + +namespace NKikimr::NOlap::NGeneralCache { + +std::shared_ptr> +TPortionsMetadataCachePolicy::BuildObjectsProcessor(const NActors::TActorId& serviceActorId) { + class TAccessorsCallback: public NDataAccessorControl::IAccessorCallback { + private: + const NActors::TActorId OwnerActorId; + const std::shared_ptr> Callback; + THashSet RequestedAddresses; + + virtual void OnAccessorsFetched(std::vector&& accessors) override { + THashMap objects; + for (auto&& i : accessors) { + const TAddress address(OwnerActorId, i.GetPortionInfo().GetAddress()); + AFL_VERIFY(RequestedAddresses.erase(address)); + objects.emplace(address, std::move(i)); + } + Callback->OnReceiveData(std::move(objects), std::move(RequestedAddresses), {}); + } + + public: + TAccessorsCallback(const NActors::TActorId& ownerActorId, + const std::shared_ptr>& callback, + THashSet&& requestedAddresses) + : OwnerActorId(ownerActorId) + , Callback(callback) + , RequestedAddresses(std::move(requestedAddresses)) { + } + }; + + class TObjectsProcessor: public NKikimr::NGeneralCache::NSource::IObjectsProcessor { + private: + using TAddress = TGlobalPortionAddress; + using TObject = TPortionDataAccessor; + using EConsumer = TPortionsMetadataCachePolicy::EConsumer; + using TSelf = NKikimr::NGeneralCache::NSource::IObjectsProcessor; + + const NActors::TActorId ServiceActorId; + + class TActorRequestData { + private: + THashMap Data; + THashSet Requested; + + public: + void Add(const EConsumer consumer, const TAddress& addr) { + Data[addr.GetPathId()].UpsertConsumer(consumer).AddPortion(addr.GetPortionId()); + AFL_VERIFY(Requested.emplace(addr).second); + } + + THashMap ExtractRequest() { + return std::move(Data); + } + THashSet ExtractRequestedAddresses() { + return std::move(Requested); + } + }; + + virtual void DoAskData( + const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const override { + THashMap requests; + for (auto&& [c, addresses] : objectAddressesByConsumer) { + for (auto&& a : addresses) { + requests[a.GetTabletActorId()].Add(c, a); + } + } + for (auto&& i : requests) { + NActors::TActivationContext::Send( + i.first, std::make_unique(i.second.ExtractRequest(), + std::make_shared(i.first, selfPtr, i.second.ExtractRequestedAddresses()))); + } + } + virtual void DoOnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + THashMap&& errors) const override { + NActors::TActivationContext::Send( + ServiceActorId, std::make_unique::TEvObjectsInfo>( + std::move(objectAddresses), std::move(removedAddresses), std::move(errors))); + } + + public: + TObjectsProcessor(const NActors::TActorId& serviceActorId) + : ServiceActorId(serviceActorId) { + } + }; + + return std::make_shared(serviceActorId); +} +} // namespace NKikimr::NOlap::NGeneralCache diff --git a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h new file mode 100644 index 000000000000..f9655a084d3e --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h @@ -0,0 +1,76 @@ +#pragma once +#include +#include +#include +#include + +#include + +namespace NKikimr::NOlap::NGeneralCache { + +class TGlobalPortionAddress { +private: + YDB_READONLY_DEF(NActors::TActorId, TabletActorId); + TPortionAddress InternalPortionAddress; + +public: + const TPortionAddress& GetInternalPortionAddress() const { + return InternalPortionAddress; + } + + ui64 GetPortionId() const { + return InternalPortionAddress.GetPortionId(); + } + + TInternalPathId GetPathId() const { + return InternalPortionAddress.GetPathId(); + } + + TGlobalPortionAddress(const NActors::TActorId& actorId, const TPortionAddress& internalAddress) + : TabletActorId(actorId) + , InternalPortionAddress(internalAddress) { + } + + bool operator==(const TGlobalPortionAddress& item) const { + return TabletActorId == item.TabletActorId && InternalPortionAddress == item.InternalPortionAddress; + } + + explicit operator size_t() const { + return TabletActorId.Hash() ^ THash()(InternalPortionAddress); + } +}; + +class TPortionsMetadataCachePolicy { +public: + using TAddress = TGlobalPortionAddress; + using TObject = TPortionDataAccessor; + using EConsumer = NOlap::NBlobOperations::EConsumer; + + static EConsumer DefaultConsumer() { + return EConsumer::UNDEFINED; + } + + class TSizeCalcer { + public: + size_t operator()(const TObject& data) { + return sizeof(TAddress) + data.GetMetadataSize(); + } + }; + + static TString GetCacheName() { + return "portions_metadata"; + } + + static TString GetServiceCode() { + return "PRMT"; + } + + static std::shared_ptr> BuildObjectsProcessor( + const NActors::TActorId& serviceActorId); +}; + +} // namespace NKikimr::NOlap::NGeneralCache + +namespace NKikimr::NOlap::NDataAccessorControl { + using TGeneralCache = NKikimr::NGeneralCache::TServiceOperator; +} diff --git a/ydb/core/tx/columnshard/data_accessor/cache_policy/ya.make b/ydb/core/tx/columnshard/data_accessor/cache_policy/ya.make new file mode 100644 index 000000000000..1a1502c0149f --- /dev/null +++ b/ydb/core/tx/columnshard/data_accessor/cache_policy/ya.make @@ -0,0 +1,11 @@ +LIBRARY() + +SRCS( + policy.cpp +) + +PEERDIR( + ydb/core/tx/columnshard/data_accessor/abstract +) + +END() diff --git a/ydb/core/tx/columnshard/data_accessor/events.cpp b/ydb/core/tx/columnshard/data_accessor/events.cpp deleted file mode 100644 index a1ec74cd603d..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/events.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "events.h" - -namespace NKikimr::NOlap::NDataAccessorControl { - -} diff --git a/ydb/core/tx/columnshard/data_accessor/events.h b/ydb/core/tx/columnshard/data_accessor/events.h deleted file mode 100644 index 11940bbd1050..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/events.h +++ /dev/null @@ -1,127 +0,0 @@ -#pragma once - -#include "abstract/collector.h" - -#include -#include -#include - -#include -#include - -namespace NKikimr::NOlap { -class IGranuleDataAccessor; -class TDataAccessorsRequest; -} // namespace NKikimr::NOlap - -namespace NKikimr::NOlap::NDataAccessorControl { - -class TEvAddPortion: public NActors::TEventLocal { -private: - std::vector Accessors; - YDB_READONLY_DEF(TActorId, Owner); - -public: - std::vector ExtractAccessors() { - return std::move(Accessors); - } - - explicit TEvAddPortion(const TPortionDataAccessor& accessor, TActorId owner) - : Accessors({accessor}) - , Owner(owner) - { - } - - explicit TEvAddPortion(const std::vector& accessors, TActorId owner) - : Accessors(accessors) - , Owner(owner) - { - } -}; - -class TEvRemovePortion: public NActors::TEventLocal { -private: - YDB_READONLY_DEF(TPortionInfo::TConstPtr, Portion); - YDB_READONLY_DEF(TActorId, Owner); - -public: - explicit TEvRemovePortion(const TPortionInfo::TConstPtr& portion, TActorId owner) - : Portion(portion) - , Owner(owner) - { - } -}; - -class TEvRegisterController: public NActors::TEventLocal { -private: - std::unique_ptr Controller; - YDB_READONLY_DEF(bool, IsUpdateFlag); - YDB_READONLY_DEF(TActorId, Owner); - - -public: - std::unique_ptr ExtractController() { - return std::move(Controller); - } - - explicit TEvRegisterController(std::unique_ptr&& accessor, const bool isUpdate, TActorId owner) - : Controller(std::move(accessor)) - , IsUpdateFlag(isUpdate) - , Owner(owner) - { - } -}; - -class TEvUnregisterController - : public NActors::TEventLocal { -private: - YDB_READONLY_DEF(TInternalPathId, PathId); - YDB_READONLY_DEF(TActorId, Owner); - -public: - explicit TEvUnregisterController(const TInternalPathId pathId, TActorId owner) - : PathId(pathId) - , Owner(owner) - { - } -}; - -class TEvClearCache - : public NActors::TEventLocal { -private: - YDB_READONLY_DEF(TActorId, Owner); - -public: - explicit TEvClearCache(TActorId owner) : Owner(owner) {} -}; - - -class TEvAskTabletDataAccessors - : public NActors::TEventLocal { -private: - using TPortions = THashMap; - YDB_ACCESSOR_DEF(TPortions, Portions); - YDB_READONLY_DEF(std::shared_ptr, Callback); - -public: - explicit TEvAskTabletDataAccessors(TPortions&& portions, const std::shared_ptr& callback) - : Portions(std::move(portions)) - , Callback(callback) { - } -}; - -class TEvAskServiceDataAccessors - : public NActors::TEventLocal { -private: - YDB_READONLY_DEF(std::shared_ptr, Request); - YDB_READONLY_DEF(TActorId, Owner); - -public: - explicit TEvAskServiceDataAccessors(const std::shared_ptr& request, TActorId owner) - : Request(request) - , Owner(owner) - { - } -}; - -} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp deleted file mode 100644 index 6916003aa375..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include "collector.h" - -namespace NKikimr::NOlap::NDataAccessorControl::NInMem { - -void TCollector::DoAskData( - THashMap&& /*portions*/, const std::shared_ptr& /*callback*/) { - AFL_VERIFY(false); -} - -TDataCategorized TCollector::DoAnalyzeData(const TPortionsByConsumer& portions) { - TDataCategorized result; - for (auto&& c : portions.GetConsumers()) { - for (auto&& p : c.second.GetPortions()) { - auto it = Accessors.find(p->GetPortionId()); - AFL_VERIFY(it != Accessors.end()); - result.AddFromCache(it->second); - } - } - return result; -} - -void TCollector::DoModifyPortions(const std::vector& add, const std::vector& remove) { - for (auto&& i : remove) { - AFL_VERIFY(Accessors.erase(i)); - } - for (auto&& i : add) { - AFL_VERIFY(Accessors.emplace(i.GetPortionInfo().GetPortionId(), i).second); - } -} - -void TCollector::DoSetCache(std::shared_ptr) { - AFL_VERIFY(false); -} - -void TCollector::DoSetOwner(const TActorId&) { - AFL_VERIFY(false); -} - -} // namespace NKikimr::NOlap::NDataAccessorControl::NInMem diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h b/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h deleted file mode 100644 index 3646e255817a..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/collector.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once -#include -#include - -namespace NKikimr::NOlap::NDataAccessorControl::NInMem { -class TCollector: public IGranuleDataAccessor { -private: - using TBase = IGranuleDataAccessor; - THashMap Accessors; - virtual void DoAskData( - THashMap&& portions, const std::shared_ptr& callback) override; - virtual TDataCategorized DoAnalyzeData(const TPortionsByConsumer& portions) override; - virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) override; - virtual void DoSetCache(std::shared_ptr) override; - virtual void DoSetOwner(const TActorId&) override; - - -public: - TCollector(const TInternalPathId pathId) - : TBase(pathId) { - } -}; - -} // namespace NKikimr::NOlap::NDataAccessorControl::NInMem diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp index c136734d172b..dde5c6fd48dd 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp @@ -1,4 +1,3 @@ -#include "collector.h" #include "manager.h" #include @@ -19,8 +18,4 @@ std::shared_ptr TManager::DoBuildLoader( return result; } -std::unique_ptr TManager::DoBuildCollector(const TInternalPathId pathId) { - return std::make_unique(pathId); -} - } // namespace NKikimr::NOlap::NDataAccessorControl::NInMem diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.h b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.h index 442c90f056a4..38e5dff62b12 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.h @@ -6,7 +6,6 @@ namespace NKikimr::NOlap::NDataAccessorControl::NInMem { class TManager: public IMetadataMemoryManager { private: - virtual std::unique_ptr DoBuildCollector(const TInternalPathId pathId) override; virtual std::shared_ptr DoBuildLoader( const TVersionedIndex& versionedIndex, TGranuleMeta* granule, const std::shared_ptr& dsGroupSelector) override; diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/ya.make b/ydb/core/tx/columnshard/data_accessor/in_mem/ya.make index d6218eca2880..9eac491959a6 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/ya.make +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/ya.make @@ -2,7 +2,6 @@ LIBRARY() SRCS( manager.cpp - collector.cpp GLOBAL constructor.cpp ) diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp b/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp deleted file mode 100644 index b451771f9dda..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/local_db/collector.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "collector.h" - -#include -namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB { - -void TCollector::DoAskData(THashMap&& portions, const std::shared_ptr& callback) { - NActors::TActivationContext::Send( - TabletActorId, std::make_unique(std::move(portions), callback)); -} - -TDataCategorized TCollector::DoAnalyzeData(const TPortionsByConsumer& portions) { - TDataCategorized result; - for (auto&& c : portions.GetConsumers()) { - TConsumerPortions* cPortions = nullptr; - for (auto&& p : c.second.GetPortions()) { - auto key = std::tuple{Owner, p->GetPathId(), p->GetPortionId()}; - if (auto it = AccessorsCache->Find(key); it != AccessorsCache->End()) { - AFL_VERIFY(it.Key() == key); - result.AddFromCache(it.Value()); - } else { - if (!cPortions) { - cPortions = &result.MutablePortionsToAsk().UpsertConsumer(c.first); - } - cPortions->AddPortion(p); - } - } - } - return result; -} - -void TCollector::DoModifyPortions(const std::vector& add, const std::vector& remove) { - for (auto&& i : remove) { - TPortionDataAccessor result = TPortionDataAccessor::BuildEmpty(); - AccessorsCache->PickOut(std::tuple{Owner, GetPathId(), i}, &result); - } - for (auto&& i : add) { - AccessorsCache->Insert(std::tuple{Owner, GetPathId(), i.GetPortionInfo().GetPortionId()}, i); - } -} - -void TCollector::DoSetCache(std::shared_ptr cache) { - AccessorsCache = cache; -} - -void TCollector::DoSetOwner(const TActorId& owner) { - Owner = TActorId(owner); -} - -} // namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/collector.h b/ydb/core/tx/columnshard/data_accessor/local_db/collector.h deleted file mode 100644 index f91d84681e94..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/local_db/collector.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once -#include -#include -#include - -#include - -namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB { - -class TCollector: public IGranuleDataAccessor { -private: - const NActors::TActorId TabletActorId; - TActorId Owner{}; - - std::shared_ptr AccessorsCache; - using TBase = IGranuleDataAccessor; - virtual void DoAskData(THashMap&& portions, - const std::shared_ptr& callback) override; - virtual TDataCategorized DoAnalyzeData(const TPortionsByConsumer& portions) override; - virtual void DoModifyPortions(const std::vector& add, const std::vector& remove) override; - virtual void DoSetCache(std::shared_ptr) override; - virtual void DoSetOwner(const TActorId& owner) override; - -public: - TCollector(const TInternalPathId pathId, const ui64 maxSize, const NActors::TActorId& actorId) - : TBase(pathId) - , TabletActorId(actorId) - { - AccessorsCache = std::make_shared(maxSize); - } -}; - -} // namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/manager.cpp b/ydb/core/tx/columnshard/data_accessor/local_db/manager.cpp index 7c80ca0fcb1a..172cd76a9e97 100644 --- a/ydb/core/tx/columnshard/data_accessor/local_db/manager.cpp +++ b/ydb/core/tx/columnshard/data_accessor/local_db/manager.cpp @@ -1,4 +1,3 @@ -#include "collector.h" #include "manager.h" #include @@ -10,8 +9,4 @@ std::shared_ptr TManager::DoBuildLoader( return nullptr; } -std::unique_ptr TManager::DoBuildCollector(const TInternalPathId pathId) { - return std::make_unique(pathId, MemoryCacheSize, TabletActorId); -} - } // namespace NKikimr::NOlap::NDataAccessorControl::NLocalDB diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/manager.h b/ydb/core/tx/columnshard/data_accessor/local_db/manager.h index ed8ad94f3a29..c1e19a144d03 100644 --- a/ydb/core/tx/columnshard/data_accessor/local_db/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/local_db/manager.h @@ -8,7 +8,6 @@ class TManager: public IMetadataMemoryManager { const NActors::TActorId TabletActorId; const ui64 MemoryCacheSize; const bool FetchOnStart = true; - virtual std::unique_ptr DoBuildCollector(const TInternalPathId pathId) override; virtual std::shared_ptr DoBuildLoader( const TVersionedIndex& versionedIndex, TGranuleMeta* granule, const std::shared_ptr& dsGroupSelector) override; diff --git a/ydb/core/tx/columnshard/data_accessor/local_db/ya.make b/ydb/core/tx/columnshard/data_accessor/local_db/ya.make index d6218eca2880..9eac491959a6 100644 --- a/ydb/core/tx/columnshard/data_accessor/local_db/ya.make +++ b/ydb/core/tx/columnshard/data_accessor/local_db/ya.make @@ -2,7 +2,6 @@ LIBRARY() SRCS( manager.cpp - collector.cpp GLOBAL constructor.cpp ) diff --git a/ydb/core/tx/columnshard/data_accessor/manager.cpp b/ydb/core/tx/columnshard/data_accessor/manager.cpp index 06d77a368c90..89b72274273c 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.cpp +++ b/ydb/core/tx/columnshard/data_accessor/manager.cpp @@ -6,131 +6,4 @@ namespace NKikimr::NOlap::NDataAccessorControl { -void TLocalManager::DrainQueue() { - std::optional lastPathId; - IGranuleDataAccessor* lastDataAccessor = nullptr; - TPositiveControlInteger countToFlight; - const ui32 inFlightLimit = NYDBTest::TControllers::GetColumnShardController()->GetLimitForPortionsMetadataAsk(); - while (PortionsAskInFlight + countToFlight < inFlightLimit && PortionsAsk.size()) { - THashMap portionsToAsk; - ui32 packPortionsCount = 0; - while (PortionsAskInFlight + countToFlight < inFlightLimit && packPortionsCount < std::min(inFlightLimit, 1000) && - PortionsAsk.size()) { - auto p = PortionsAsk.front().ExtractPortion(); - const TString consumerId = PortionsAsk.front().GetConsumerId(); - PortionsAsk.pop_front(); - if (!lastPathId || *lastPathId != p->GetPathId()) { - lastPathId = p->GetPathId(); - auto it = Managers.find(p->GetPathId()); - if (it == Managers.end()) { - lastDataAccessor = nullptr; - } else { - lastDataAccessor = it->second.get(); - } - } - auto it = RequestsByPortion.find(p->GetPortionId()); - if (it == RequestsByPortion.end()) { - continue; - } - if (!lastDataAccessor) { - for (auto&& i : it->second) { - if (!i->IsFetched() && !i->IsAborted()) { - i->AddError(p->GetPathId(), "path id absent"); - } - } - RequestsByPortion.erase(it); - } else { - bool toAsk = false; - for (auto&& i : it->second) { - if (!i->IsFetched() && !i->IsAborted()) { - toAsk = true; - } - } - if (!toAsk) { - RequestsByPortion.erase(it); - } else { - portionsToAsk[p->GetPathId()].UpsertConsumer(consumerId).AddPortion(p); - ++packPortionsCount; - ++countToFlight; - } - } - } - for (auto&& i : portionsToAsk) { - auto it = Managers.find(i.first); - AFL_VERIFY(it != Managers.end()); - auto dataAnalyzed = it->second->AnalyzeData(i.second); - for (auto&& accessor : dataAnalyzed.GetCachedAccessors()) { - auto it = RequestsByPortion.find(accessor.GetPortionInfo().GetPortionId()); - AFL_VERIFY(it != RequestsByPortion.end()); - for (auto&& i : it->second) { - Counters.ResultFromCache->Add(1); - if (!i->IsFetched() && !i->IsAborted()) { - i->AddAccessor(accessor); - } - } - RequestsByPortion.erase(it); - --countToFlight; - } - if (!dataAnalyzed.GetPortionsToAsk().IsEmpty()) { - THashMap portionsToAskImpl; - Counters.ResultAskDirectly->Add(dataAnalyzed.GetPortionsToAsk().GetPortionsCount()); - portionsToAskImpl.emplace(i.first, dataAnalyzed.DetachPortionsToAsk()); - it->second->AskData(std::move(portionsToAskImpl), AccessorCallback); - } - } - } - PortionsAskInFlight.Add(countToFlight); - Counters.FetchingCount->Set(PortionsAskInFlight); - Counters.QueueSize->Set(PortionsAsk.size()); -} - -void TLocalManager::DoAskData(const std::shared_ptr& request) { - AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "ask_data")("request", request->DebugString()); - for (auto&& pathId : request->GetPathIds()) { - auto portions = request->StartFetching(pathId); - for (auto&& [_, i] : portions) { - auto itRequest = RequestsByPortion.find(i->GetPortionId()); - if (itRequest == RequestsByPortion.end()) { - AFL_VERIFY(RequestsByPortion.emplace(i->GetPortionId(), std::vector>({request})).second); - PortionsAsk.emplace_back(i, request->GetAbortionFlag(), request->GetConsumer()); - Counters.AskNew->Add(1); - } else { - itRequest->second.emplace_back(request); - Counters.AskDuplication->Add(1); - } - } - } - DrainQueue(); -} - -void TLocalManager::DoRegisterController(std::unique_ptr&& controller, const bool update) { - if (update) { - auto it = Managers.find(controller->GetPathId()); - if (it != Managers.end()) { - it->second = std::move(controller); - } - } else { - AFL_VERIFY(Managers.emplace(controller->GetPathId(), std::move(controller)).second); - } -} - -void TLocalManager::DoAddPortion(const TPortionDataAccessor& accessor) { - { - auto it = Managers.find(accessor.GetPortionInfo().GetPathId()); - AFL_VERIFY(it != Managers.end()); - it->second->ModifyPortions({ accessor }, {}); - } - { - auto it = RequestsByPortion.find(accessor.GetPortionInfo().GetPortionId()); - if (it != RequestsByPortion.end()) { - for (auto&& i : it->second) { - i->AddAccessor(accessor); - } - --PortionsAskInFlight; - } - RequestsByPortion.erase(it); - } - DrainQueue(); -} - } // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/manager.h b/ydb/core/tx/columnshard/data_accessor/manager.h index 781c80118b99..573fe1e4d6af 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/manager.h @@ -1,43 +1,19 @@ #pragma once -#include "events.h" #include "request.h" #include "abstract/collector.h" #include +#include +#include #include namespace NKikimr::NOlap::NDataAccessorControl { -class TAccessorSignals: public NColumnShard::TCommonCountersOwner { -private: - using TBase = NColumnShard::TCommonCountersOwner; - -public: - const NMonitoring::TDynamicCounters::TCounterPtr QueueSize; - const NMonitoring::TDynamicCounters::TCounterPtr FetchingCount; - const NMonitoring::TDynamicCounters::TCounterPtr AskNew; - const NMonitoring::TDynamicCounters::TCounterPtr AskDuplication; - const NMonitoring::TDynamicCounters::TCounterPtr ResultFromCache; - const NMonitoring::TDynamicCounters::TCounterPtr ResultAskDirectly; - - TAccessorSignals() - : TBase("AccessorsFetching") - , QueueSize(TBase::GetValue("Queue/Count")) - , FetchingCount(TBase::GetValue("Fetching/Count")) - , AskNew(TBase::GetDeriviative("Ask/Fault/Count")) - , AskDuplication(TBase::GetDeriviative("Ask/Duplication/Count")) - , ResultFromCache(TBase::GetDeriviative("ResultFromCache/Count")) - , ResultAskDirectly(TBase::GetDeriviative("ResultAskDirectly/Count")) { - } -}; - class IDataAccessorsManager { private: virtual void DoAskData(const std::shared_ptr& request) = 0; - virtual void DoRegisterController(std::unique_ptr&& controller, const bool update) = 0; - virtual void DoUnregisterController(const TInternalPathId pathId) = 0; virtual void DoAddPortion(const TPortionDataAccessor& accessor) = 0; virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) = 0; const NActors::TActorId TabletActorId; @@ -64,13 +40,6 @@ class IDataAccessorsManager { AFL_VERIFY(request->HasSubscriber()); return DoAskData(request); } - void RegisterController(std::unique_ptr&& controller, const bool update) { - AFL_VERIFY(controller); - return DoRegisterController(std::move(controller), update); - } - void UnregisterController(const TInternalPathId pathId) { - return DoUnregisterController(pathId); - } }; class TDataAccessorsManagerContainer: public NBackgroundTasks::TControlInterfaceContainer { @@ -84,108 +53,54 @@ class TDataAccessorsManagerContainer: public NBackgroundTasks::TControlInterface class TActorAccessorsManager: public IDataAccessorsManager { private: using TBase = IDataAccessorsManager; - const NActors::TActorId ActorId; - std::shared_ptr AccessorsCallback; virtual void DoAskData(const std::shared_ptr& request) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(request, GetTabletActorId())); - } - virtual void DoRegisterController(std::unique_ptr&& controller, const bool update) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(std::move(controller), update, GetTabletActorId())); - } - virtual void DoUnregisterController(const TInternalPathId pathId) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(pathId, GetTabletActorId())); + class TAdapterCallback: public NKikimr::NGeneralCache::NPublic::ICallback { + private: + std::shared_ptr AccessorsCallback; + const ui64 RequestId; + virtual void DoOnResultReady(THashMap&& objectAddresses, + THashSet&& removedAddresses, + THashMap&& errorAddresses) const override { + AFL_VERIFY(removedAddresses.empty()); + AFL_VERIFY(errorAddresses.empty()); + THashMap objects; + for (auto&& i : objectAddresses) { + objects.emplace(i.first.GetPortionId(), std::move(i.second)); + } + TDataAccessorsResult result; + result.AddData(std::move(objects)); + AccessorsCallback->OnResult(RequestId, std::move(result)); + } + + public: + TAdapterCallback(const std::shared_ptr& accCallback, const ui64 requestId) + : AccessorsCallback(accCallback) + , RequestId(requestId) { + } + }; + + NKikimr::NGeneralCache::TServiceOperator::AskObjects(request->GetConsumer(), + request->BuildAddresses(GetTabletActorId()), + std::make_shared(request->ExtractSubscriber(), request->GetRequestId())); } virtual void DoAddPortion(const TPortionDataAccessor& accessor) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(accessor, GetTabletActorId())); + THashMap add; + THashSet remove; + add.emplace(NGeneralCache::TGlobalPortionAddress(GetTabletActorId(), accessor.GetPortionInfo().GetAddress()), accessor); + NKikimr::NGeneralCache::TServiceOperator::ModifyObjects(std::move(add), std::move(remove)); } virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) override { - NActors::TActivationContext::Send(ActorId, std::make_unique(portion, GetTabletActorId())); + THashMap add; + THashSet remove; + remove.emplace(NGeneralCache::TGlobalPortionAddress(GetTabletActorId(), portion->GetAddress())); + NKikimr::NGeneralCache::TServiceOperator::ModifyObjects(std::move(add), std::move(remove)); } public: - TActorAccessorsManager(const NActors::TActorId& actorId, const NActors::TActorId& tabletActorId) - : TBase(tabletActorId) - , ActorId(actorId) - , AccessorsCallback(std::make_shared(ActorId)) { + TActorAccessorsManager(const NActors::TActorId& tabletActorId) + : TBase(tabletActorId) { AFL_VERIFY(!!tabletActorId); } }; -class TLocalManager: public IDataAccessorsManager { -private: - using TBase = IDataAccessorsManager; - THashMap> Managers; - THashMap>> RequestsByPortion; - TAccessorSignals Counters; - const std::shared_ptr AccessorCallback; - - class TPortionToAsk { - private: - TPortionInfo::TConstPtr Portion; - YDB_READONLY_DEF(std::shared_ptr, AbortionFlag); - YDB_READONLY_DEF(TString, ConsumerId); - - public: - TPortionToAsk( - const TPortionInfo::TConstPtr& portion, const std::shared_ptr& abortionFlag, const TString& consumerId) - : Portion(portion) - , AbortionFlag(abortionFlag) - , ConsumerId(consumerId) { - } - - TPortionInfo::TConstPtr ExtractPortion() { - return std::move(Portion); - } - }; - - std::deque PortionsAsk; - TPositiveControlInteger PortionsAskInFlight; - - void DrainQueue(); - - virtual void DoAskData(const std::shared_ptr& request) override; - virtual void DoRegisterController(std::unique_ptr&& controller, const bool update) override; - virtual void DoUnregisterController(const TInternalPathId pathId) override { - AFL_VERIFY(Managers.erase(pathId)); - } - virtual void DoAddPortion(const TPortionDataAccessor& accessor) override; - virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portionInfo) override { - auto it = Managers.find(portionInfo->GetPathId()); - AFL_VERIFY(it != Managers.end()); - it->second->ModifyPortions({}, { portionInfo->GetPortionId() }); - } - -public: - class TTestingCallback: public IAccessorCallback { - private: - std::weak_ptr Manager; - virtual void OnAccessorsFetched(std::vector&& accessors) override { - auto mImpl = Manager.lock(); - if (!mImpl) { - return; - } - for (auto&& i : accessors) { - mImpl->AddPortion(i); - } - } - - public: - void InitManager(const std::weak_ptr& manager) { - Manager = manager; - } - }; - - static std::shared_ptr BuildForTests() { - auto callback = std::make_shared(); - std::shared_ptr result = std::make_shared(callback); - callback->InitManager(result); - return result; - } - - TLocalManager(const std::shared_ptr& callback) - : TBase(NActors::TActorId()) - , AccessorCallback(callback) { - } -}; - } // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/request.h b/ydb/core/tx/columnshard/data_accessor/request.h index 2b52a5cc4c98..7d1852e96279 100644 --- a/ydb/core/tx/columnshard/data_accessor/request.h +++ b/ydb/core/tx/columnshard/data_accessor/request.h @@ -1,9 +1,12 @@ #pragma once -#include +#include "cache_policy/policy.h" + +#include #include #include #include -#include + +#include namespace NKikimr::NOlap { @@ -93,7 +96,7 @@ class IDataAccessorRequestsSubscriber: public NColumnShard::TMonitoringObjectsCo std::optional Result; public: - void OnResult(const ui32 requestId, TDataAccessorsResult&& result) { + void OnResult(const ui64 requestId, TDataAccessorsResult&& result) { AFL_VERIFY(RequestIds.erase(requestId)); if (!Result) { Result = std::move(result); @@ -194,7 +197,8 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter PortionIds; THashMap PathIdStatus; THashSet PathIds; @@ -220,6 +224,21 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter ExtractSubscriber() { + AFL_VERIFY(HasSubscriber()); + return std::move(Subscriber); + } + + THashSet BuildAddresses(const NActors::TActorId tabletActorId) const { + THashSet result; + for (auto&& i : PathIdStatus) { + for (auto&& [_, p] : i.second.GetPortions()) { + AFL_VERIFY(result.emplace(NGeneralCache::TGlobalPortionAddress(tabletActorId, p->GetAddress())).second); + } + } + return result; + } + void SetColumnIds(const std::set& columnIds) { AFL_VERIFY(!ColumnIds); ColumnIds = columnIds; @@ -234,10 +253,8 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCountersecond.AddPortion(portion); } diff --git a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp deleted file mode 100644 index b4c4cdf5db09..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "shared_metadata_accessor_cache_actor.h" - -#include - -namespace NKikimr::NOlap::NDataAccessorControl { - -NActors::IActor* TSharedMetadataAccessorCacheActor::CreateActor() { - return new TSharedMetadataAccessorCacheActor(); -} - -void TSharedMetadataAccessorCacheActor::Bootstrap() { - AccessorsCallback = std::make_shared(SelfId()); - if (HasAppData()) { - TotalMemorySize = AppDataVerified().ColumnShardConfig.GetSharedMetadataAccessorCacheSize(); - } - MetadataCache = std::make_shared(TotalMemorySize); - Become(&TThis::StateWait); -} - -} diff --git a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h b/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h deleted file mode 100644 index da61a74cb340..000000000000 --- a/ydb/core/tx/columnshard/data_accessor/shared_metadata_accessor_cache_actor.h +++ /dev/null @@ -1,113 +0,0 @@ -#pragma once -#include "events.h" -#include "manager.h" - -#include -#include -#include - -namespace NKikimr::NOlap::NDataAccessorControl { -class TSharedMetadataAccessorCacheActor: public TActorBootstrapped { -private: - ui64 TotalMemorySize = 1 << 30; - THashMap Managers; - std::shared_ptr MetadataCache; - std::shared_ptr AccessorsCallback; - - void StartStopping() { - AFL_VERIFY(false)("shared_metadata_accessor_cache", "trying to call StartStopping()"); - PassAway(); - } - - void Handle(TEvRegisterController::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvRegisterController")("owner", ev->Get()->GetOwner()); - auto controller = ev->Get()->ExtractController(); - auto owner = ev->Get()->GetOwner(); - controller->SetCache(MetadataCache); - controller->SetOwner(owner); - auto manager = Managers.find(owner); - if (manager == Managers.end()) { - manager = Managers.emplace(owner, std::make_shared(AccessorsCallback, owner)).first; - } - manager->second.RegisterController(move(controller), ev->Get()->GetIsUpdateFlag()); - } - - void Handle(TEvUnregisterController::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvUnregisterController")("owner", ev->Get()->GetOwner()); - if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { - manager->second.UnregisterController(ev->Get()->GetPathId()); - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); - } - } - - void Handle(TEvAddPortion::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvAddPortion")("owner", ev->Get()->GetOwner()); - if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { - for (auto&& a : ev->Get()->ExtractAccessors()) { - manager->second.AddPortion(std::move(a)); - } - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); - } - } - - void Handle(TEvRemovePortion::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvRemovePortion")("owner", ev->Get()->GetOwner()); - if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { - manager->second.RemovePortion(ev->Get()->GetPortion()); - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); - } - } - - void Handle(TEvAskServiceDataAccessors::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvAskServiceDataAccessors")("owner", ev->Get()->GetOwner()); - if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { - manager->second.AskData(ev->Get()->GetRequest()); - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); - } - } - - void Handle(TEvClearCache::TPtr& ev) { - AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "TEvClearCache")("owner", ev->Get()->GetOwner()); - if (auto manager = Managers.find(ev->Get()->GetOwner()); manager != Managers.end()) { - Managers.erase(manager); - } else { - AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("shared_metadata_accessor_cache", "owner_not_found"); - } - } - -public: - - static inline TActorId MakeActorId(ui32 nodeId) { - char x[12] = {'s', 'h', 'a', 'r', 'e', - 'd', 'm', 'e', 't', 'a', 'd', 't'}; - return TActorId(nodeId, TStringBuf(x, 12)); - } - - static NActors::IActor* CreateActor(); - - TSharedMetadataAccessorCacheActor() = default; - ~TSharedMetadataAccessorCacheActor() = default; - - void Bootstrap(); - - STFUNC(StateWait) { - const NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("self_id", SelfId()); - switch (ev->GetTypeRewrite()) { - cFunc(NActors::TEvents::TEvPoison::EventType, StartStopping); - hFunc(TEvRegisterController, Handle); - hFunc(TEvUnregisterController, Handle); - hFunc(TEvAskServiceDataAccessors, Handle); - hFunc(TEvRemovePortion, Handle); - hFunc(TEvAddPortion, Handle); - hFunc(TEvClearCache, Handle); - default: - AFL_VERIFY(false); - } - } -}; - -} // namespace NKikimr::NOlap::NDataAccessorControl diff --git a/ydb/core/tx/columnshard/data_accessor/ya.make b/ydb/core/tx/columnshard/data_accessor/ya.make index b255d527d457..5b7f12591faf 100644 --- a/ydb/core/tx/columnshard/data_accessor/ya.make +++ b/ydb/core/tx/columnshard/data_accessor/ya.make @@ -1,11 +1,8 @@ LIBRARY() SRCS( - actor.cpp - events.cpp - manager.cpp request.cpp - shared_metadata_accessor_cache_actor.cpp + manager.cpp ) PEERDIR( @@ -13,6 +10,7 @@ PEERDIR( ydb/core/tx/columnshard/engines/portions ydb/core/tx/columnshard/data_accessor/abstract ydb/core/tx/columnshard/data_accessor/local_db + ydb/core/tx/columnshard/data_accessor/cache_policy ydb/core/tx/columnshard/resource_subscriber ) diff --git a/ydb/core/tx/columnshard/data_reader/contexts.cpp b/ydb/core/tx/columnshard/data_reader/contexts.cpp index f5ffdc7afebc..ff1959551356 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.cpp +++ b/ydb/core/tx/columnshard/data_reader/contexts.cpp @@ -1,6 +1,8 @@ #include "contexts.h" #include "fetcher.h" +#include + namespace NKikimr::NOlap::NDataFetcher { IFetchingStep::EStepResult IFetchingStep::Execute(const std::shared_ptr& fetchingContext) const { @@ -11,4 +13,15 @@ IFetchingStep::EStepResult IFetchingStep::Execute(const std::shared_ptr& portions, const std::shared_ptr& versions, + const NBlobOperations::EConsumer consumer, const TString& externalTaskId) + : Consumer(consumer) + , ExternalTaskId(externalTaskId) { + AFL_VERIFY(portions.size()); + ActualSchema = versions->GetLastSchema(); + for (auto&& i : portions) { + Portions.emplace_back(std::make_shared(i, versions->GetSchemaVerified(i->GetSchemaVersionVerified()))); + } +} + } // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h index 259998dba5cc..f34122447775 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.h +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -249,15 +249,7 @@ class TRequestInput { public: TRequestInput(const std::vector& portions, const std::shared_ptr& versions, - const NBlobOperations::EConsumer consumer, const TString& externalTaskId) - : Consumer(consumer) - , ExternalTaskId(externalTaskId) { - AFL_VERIFY(portions.size()); - ActualSchema = versions->GetLastSchema(); - for (auto&& i : portions) { - Portions.emplace_back(std::make_shared(i, versions->GetSchemaVerified(i->GetSchemaVersionVerified()))); - } - } + const NBlobOperations::EConsumer consumer, const TString& externalTaskId); }; } // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetching_steps.h b/ydb/core/tx/columnshard/data_reader/fetching_steps.h index 1f01404a03a1..1659961e439d 100644 --- a/ydb/core/tx/columnshard/data_reader/fetching_steps.h +++ b/ydb/core/tx/columnshard/data_reader/fetching_steps.h @@ -40,7 +40,7 @@ class TAskAccessorResourcesStep: public IFetchingStep { virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { fetchingContext->SetStage(EFetchingStage::AskAccessorResources); - auto request = std::make_shared(::ToString(fetchingContext->GetInput().GetConsumer())); + auto request = std::make_shared(fetchingContext->GetInput().GetConsumer()); for (auto&& i : fetchingContext->GetInput().GetPortions()) { request->AddPortion(i->GetPortionInfo()); } @@ -89,7 +89,7 @@ class TAskAccessorsStep: public IFetchingStep { virtual IFetchingStep::EStepResult DoExecute(const std::shared_ptr& fetchingContext) const override { fetchingContext->SetStage(EFetchingStage::AskAccessors); std::shared_ptr request = - std::make_shared(::ToString(fetchingContext->GetInput().GetConsumer())); + std::make_shared(fetchingContext->GetInput().GetConsumer()); request->RegisterSubscriber(std::make_shared(fetchingContext)); for (auto&& i : fetchingContext->GetInput().GetPortions()) { request->AddPortion(i->GetPortionInfo()); diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index 6bf35c6dd29e..916c198a2bbb 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -285,7 +285,8 @@ class TColumnEngineChanges: public TMoveOnly { return DoBuildDataLock(); } - std::shared_ptr PortionsToAccess = std::make_shared(TaskIdentifier); + std::shared_ptr PortionsToAccess = + std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::GENERAL_COMPACTION); virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& context) = 0; public: diff --git a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h index b8b0c19f782c..8ab74c562623 100644 --- a/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h +++ b/ydb/core/tx/columnshard/engines/reader/abstract/read_context.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 664a10159c40..4c259b235b7d 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -191,7 +191,8 @@ bool TPortionDataSource::DoStartFetchingAccessor(const std::shared_ptr request = std::make_shared("PLAIN::" + step.GetName()); + std::shared_ptr request = + std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::SCAN); request->AddPortion(Portion); request->RegisterSubscriber(std::make_shared(step, sourcePtr)); GetContext()->GetCommonContext()->GetDataAccessorsManager()->AskData(request); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index e302537fd4ff..a20d9395a5bf 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -417,7 +417,7 @@ bool TPortionDataSource::DoStartFetchingAccessor(const std::shared_ptr request = std::make_shared("SIMPLE::" + step.GetName()); + std::shared_ptr request = std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::SCAN); request->AddPortion(Portion); request->SetColumnIds(GetContext()->GetAllUsageColumns()->GetColumnIds()); request->RegisterSubscriber(std::make_shared(step, sourcePtr)); diff --git a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp index 976a9ec8e8f8..cf00df2e44fc 100644 --- a/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp +++ b/ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp @@ -298,7 +298,7 @@ std::vector TTieringActualizer::BuildMetadataRequests( std::shared_ptr currentRequest; for (auto&& i : NewPortionIds) { if (!currentRequest) { - currentRequest = std::make_shared("TIERING_ACTUALIZER"); + currentRequest = std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::TTL); } auto it = portions.find(i); AFL_VERIFY(it != portions.end()); diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp index 77be9bd4d170..f18e8b876fb1 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp @@ -155,6 +155,7 @@ void TGranuleMeta::UpsertPortionOnLoad(const std::shared_ptr& port AtomicSet(LastInsertWriteId, (i64)portionImpl->GetInsertWriteId()); } AFL_VERIFY(InsertedPortions.emplace(insertWriteId, portionImpl).second); + AFL_VERIFY(InsertedPortionsById.emplace(portionImpl->GetPortionId(), portionImpl).second); AFL_VERIFY(!Portions.contains(portionImpl->GetPortionId())); } else { auto portionId = portion->GetPortionId(); @@ -175,7 +176,6 @@ void TGranuleMeta::BuildActualizationTasks(NActualizer::TTieringProcessContext& void TGranuleMeta::ResetAccessorsManager(const std::shared_ptr& constructor, const NDataAccessorControl::TManagerConstructionContext& context) { MetadataMemoryManager = constructor->Build(context).DetachResult(); - DataAccessorsManager->RegisterController(MetadataMemoryManager->BuildCollector(PathId), context.IsUpdate()); } void TGranuleMeta::ResetOptimizer(const std::shared_ptr& constructor, @@ -289,6 +289,7 @@ void TGranuleMeta::InsertPortionOnComplete(const TPortionDataAccessor& portion, AFL_VERIFY(portionImpl->GetPortionType() == EPortionType::Written); auto writtenPortion = std::static_pointer_cast(portionImpl); AFL_VERIFY(InsertedPortions.emplace(writtenPortion->GetInsertWriteId(), writtenPortion).second); + AFL_VERIFY(InsertedPortionsById.emplace(portionImpl->GetPortionId(), writtenPortion).second); AFL_VERIFY(InsertedAccessors.emplace(writtenPortion->GetInsertWriteId(), portion).second); DataAccessorsManager->AddPortion(portion); } @@ -305,6 +306,7 @@ void TGranuleMeta::CommitPortionOnExecute( void TGranuleMeta::CommitPortionOnComplete(const TInsertWriteId insertWriteId, IColumnEngine& engine) { auto it = InsertedPortions.find(insertWriteId); AFL_VERIFY(it != InsertedPortions.end()); + AFL_VERIFY(InsertedPortionsById.erase(it->second->GetPortionId())); (static_cast(&engine))->AppendPortion(it->second); InsertedPortions.erase(it); { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index a323ef34ba9a..11b0c79cd859 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -121,6 +121,7 @@ class TGranuleMeta: TNonCopyable { THashMap> Portions; TAtomic LastInsertWriteId = 1; THashMap> InsertedPortions; + THashMap> InsertedPortionsById; THashMap InsertedAccessors; mutable std::optional AdditiveSummaryCache; @@ -182,12 +183,6 @@ class TGranuleMeta: TNonCopyable { return DataAccessorsManager; } - std::unique_ptr BuildDataAccessor() { - AFL_VERIFY(!DataAccessorConstructed); - DataAccessorConstructed = true; - return MetadataMemoryManager->BuildCollector(PathId); - } - void RefreshTiering(const std::optional& tiering) { NActualizer::TAddExternalContext context(HasAppData() ? AppDataVerified().TimeProvider->Now() : TInstant::Now(), Portions); ActualizationIndex->RefreshTiering(tiering, context); @@ -229,6 +224,7 @@ class TGranuleMeta: TNonCopyable { NTabletFlatExecutor::TTransactionContext& txc, const TInsertWriteId insertWriteId, const TSnapshot ssRemove) const { auto it = InsertedPortions.find(insertWriteId); AFL_VERIFY(it != InsertedPortions.end()); + AFL_VERIFY(InsertedPortionsById.contains(it->second->GetPortionId())); it->second->SetCommitSnapshot(ssRemove); it->second->SetRemoveSnapshot(ssRemove); TDbWrapper wrapper(txc.DB, nullptr); @@ -310,7 +306,7 @@ class TGranuleMeta: TNonCopyable { OnAfterChangePortion(i.second, &g, true); } if (MetadataMemoryManager->NeedPrefetch() && Portions.size()) { - auto request = std::make_shared("PREFETCH_GRANULE::" + ::ToString(PathId)); + auto request = std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::FETCH_ON_LOAD); for (auto&& p : Portions) { request->AddPortion(p.second); } @@ -365,6 +361,12 @@ class TGranuleMeta: TNonCopyable { return InsertedPortions; } + const std::shared_ptr& GetInsertedPortionVerifiedPtr(const TInsertWriteId portionId) const { + auto it = InsertedPortions.find(portionId); + AFL_VERIFY(it != InsertedPortions.end()); + return it->second; + } + std::vector> GetPortionsVector() const { std::vector> result; for (auto&& i : Portions) { @@ -383,6 +385,21 @@ class TGranuleMeta: TNonCopyable { return *it->second; } + TPortionInfo::TPtr GetPortionVerifiedPtr(const ui64 portion, const bool committedOnly = true) const { + { + auto it = Portions.find(portion); + if (it != Portions.end()) { + return it->second; + } + } + AFL_VERIFY(!committedOnly); + { + auto it = InsertedPortionsById.find(portion); + AFL_VERIFY(it != InsertedPortionsById.end()); + return it->second; + } + } + std::shared_ptr GetPortionOptional(const ui64 portion) const { auto it = Portions.find(portion); if (it == Portions.end()) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/storage.h b/ydb/core/tx/columnshard/engines/storage/granule/storage.h index b38b081f31b1..eeb7cb7763c9 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/storage.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/storage.h @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -153,7 +152,6 @@ class TGranulesStorage { if (!it->second->IsErasable()) { return false; } - DataAccessorsManager->UnregisterController(pathId); Tables.erase(it); return true; } diff --git a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h index b1e4f574feb7..5496ac67faa6 100644 --- a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h +++ b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -268,6 +269,7 @@ class TNormalizationController { private: std::shared_ptr StoragesManager; + NDataAccessorControl::TDataAccessorsManagerContainer DataAccessorsManager; NOlap::NResourceBroker::NSubscribe::TTaskContext TaskSubscription; std::deque Normalizers; @@ -284,6 +286,17 @@ class TNormalizationController { const std::shared_ptr& counters) : StoragesManager(storagesManager) , TaskSubscription("CS::NORMALIZER", counters) { + AFL_VERIFY(StoragesManager); + } + + void SetDataAccessorsManager(const NDataAccessorControl::TDataAccessorsManagerContainer& dataAccessorsManager) { + AFL_VERIFY(!DataAccessorsManager); + AFL_VERIFY(!!dataAccessorsManager); + DataAccessorsManager = dataAccessorsManager; + } + + std::shared_ptr GetDataAccessorsManager() const { + return DataAccessorsManager.GetObjectPtrVerified(); } const NOlap::NResourceBroker::NSubscribe::TTaskContext& GetTaskSubscription() const { diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index 1d7e9656cd43..da28172bc22c 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -140,8 +140,8 @@ TConclusion> TChunksNormalizer::DoInit( return tasks; } - TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(nullptr), - std::make_shared(), std::make_shared(), 0); + TTablesManager tablesManager( + controller.GetStoragesManager(), controller.GetDataAccessorsManager(), std::make_shared(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TChunksNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks_v0_meta.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks_v0_meta.cpp index 6de63b7cd6fc..358aa1a3950d 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks_v0_meta.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks_v0_meta.cpp @@ -62,7 +62,7 @@ TConclusion> TChunksV0MetaNormalizer::DoInit( return TConclusionStatus::Fail("Not ready"); } - TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(nullptr), + TTablesManager tablesManager(controller.GetStoragesManager(), controller.GetDataAccessorsManager(), std::make_shared(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TChunksV0MetaNormalizer")("error", "can't initialize tables manager"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp index 61989e46e57c..8fea108e9f88 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp @@ -182,8 +182,8 @@ TConclusion> TLeakedBlobsNormalizer::DoInit( return TConclusionStatus::Fail("Not ready"); } - NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(nullptr), - std::make_shared(), std::make_shared(), TabletId); + NColumnShard::TTablesManager tablesManager( + controller.GetStoragesManager(), controller.GetDataAccessorsManager(), std::make_shared(), std::make_shared(), TabletId); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TPortionsNormalizer")("error", "can't initialize tables manager"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp index 0005d193f674..04b7ad481fe5 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp @@ -25,7 +25,7 @@ TConclusion> TPortionsNormalizerBase::DoInit( return TConclusionStatus::Fail("Not ready"); } - NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), std::make_shared(nullptr), + NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), controller.GetDataAccessorsManager(), std::make_shared(), std::make_shared(), 0); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TPortionsNormalizer")("error", "can't initialize tables manager"); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 09ce2f71e4e2..4a53cb8f56e1 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -201,7 +201,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { NOlap::IColumnEngine::TSchemaInitializationData schemaInitializationData(info); if (!PrimaryIndex) { PrimaryIndex = std::make_unique(TabletId, SchemaObjectsCache.GetObjectPtrVerified(), - DataAccessorsManager, StoragesManager, version, preset->Id, schemaInitializationData, PortionsStats); + DataAccessorsManager.GetObjectPtrVerified(), StoragesManager, version, preset->Id, schemaInitializationData, PortionsStats); } else if (PrimaryIndex->GetVersionedIndex().IsEmpty() || info.GetSchema().GetVersion() > PrimaryIndex->GetVersionedIndex().GetLastSchema()->GetVersion()) { PrimaryIndex->RegisterSchemaVersion(version, preset->Id, schemaInitializationData); @@ -343,8 +343,9 @@ void TTablesManager::AddSchemaVersion( versionInfo.MutableSchema()->SetEngine(NKikimrSchemeOp::COLUMN_ENGINE_REPLACING_TIMESERIES); Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); if (!PrimaryIndex) { - PrimaryIndex = std::make_unique(TabletId, SchemaObjectsCache.GetObjectPtrVerified(), DataAccessorsManager, - StoragesManager, version, presetId, NOlap::IColumnEngine::TSchemaInitializationData(versionInfo), PortionsStats); + PrimaryIndex = std::make_unique(TabletId, SchemaObjectsCache.GetObjectPtrVerified(), + DataAccessorsManager.GetObjectPtrVerified(), StoragesManager, version, presetId, + NOlap::IColumnEngine::TSchemaInitializationData(versionInfo), PortionsStats); for (auto&& i : Tables) { PrimaryIndex->RegisterTable(i.first); } diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 3de1ee020ac9..e56d31703ab2 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -211,7 +211,7 @@ class TTablesManager: public NOlap::IPathIdTranslator { TTtlVersions Ttl; std::unique_ptr PrimaryIndex; std::shared_ptr StoragesManager; - std::shared_ptr DataAccessorsManager; + NOlap::NDataAccessorControl::TDataAccessorsManagerContainer DataAccessorsManager; std::unique_ptr LoadTimeCounters; NBackgroundTasks::TControlInterfaceContainer SchemaObjectsCache; std::shared_ptr PortionsStats; diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp index 22f00cd0794d..7aa4ae2964a5 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include @@ -21,14 +20,9 @@ namespace NKikimr::NTxUT { using namespace NColumnShard; using namespace Tests; -void SetupSharedMetadataAccessorCacheService(TTestActorRuntime& runtime, ui32 nodeIndex) { - runtime.AddLocalService(NKikimr::NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::MakeActorId(runtime.GetNodeId(nodeIndex)), - TActorSetupCmd(NKikimr::NOlap::NDataAccessorControl::TSharedMetadataAccessorCacheActor::CreateActor(), TMailboxType::HTSwap, 0), nodeIndex); -} - void TTester::Setup(TTestActorRuntime& runtime) { runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); -// runtime.SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_INFO); + // runtime.SetLogPriority(NKikimrServices::BLOB_CACHE, NActors::NLog::PRI_INFO); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD_SCAN, NActors::NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD_WRITE, NActors::NLog::PRI_DEBUG); // runtime.SetLogPriority(NKikimrServices::S3_WRAPPER, NLog::PRI_DEBUG); @@ -42,21 +36,15 @@ void TTester::Setup(TTestActorRuntime& runtime) { TAppPrepare app; - auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds( - "dc-1", domainId, FAKE_SCHEMESHARD_TABLET_ID, - planResolution, - std::vector{TDomainsInfo::MakeTxCoordinatorIDFixed(1)}, - std::vector{}, - std::vector{TDomainsInfo::MakeTxAllocatorIDFixed(1)}); + auto domain = TDomainsInfo::TDomain::ConstructDomainWithExplicitTabletIds("dc-1", domainId, FAKE_SCHEMESHARD_TABLET_ID, planResolution, + std::vector{ TDomainsInfo::MakeTxCoordinatorIDFixed(1) }, std::vector{}, + std::vector{ TDomainsInfo::MakeTxAllocatorIDFixed(1) }); TVector ids = runtime.GetTxAllocatorTabletIds(); ids.insert(ids.end(), domain->TxAllocators.begin(), domain->TxAllocators.end()); runtime.SetTxAllocatorTabletIds(ids); app.AddDomain(domain.Release()); - for (ui32 nodeIndex = 0; nodeIndex < runtime.GetNodeCount(); ++nodeIndex) { - SetupSharedMetadataAccessorCacheService(runtime, nodeIndex); - } SetupTabletServices(runtime, &app); runtime.UpdateCurrentTime(TInstant::Now()); @@ -70,8 +58,7 @@ void RefreshTiering(TTestBasicRuntime& runtime, const TActorId& sender) { namespace { std::optional ProposeSchemaTxOptional(TTestBasicRuntime& runtime, TActorId& sender, const TString& txBody, const ui64 txId) { - auto event = std::make_unique( - NKikimrTxColumnShard::TX_KIND_SCHEMA, 0, sender, txId, txBody, 0, 0); + auto event = std::make_unique(NKikimrTxColumnShard::TX_KIND_SCHEMA, 0, sender, txId, txBody, 0, 0); const auto now = runtime.GetTimeProvider()->Now(); ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, event.release()); auto ev = runtime.GrabEdgeEvent(sender); @@ -81,11 +68,11 @@ std::optional ProposeSchemaTxOptional(TTestBasicRuntime& runtime, TAc if (res.GetStatus() == NKikimrTxColumnShard::PREPARED) { UNIT_ASSERT_LE(now.MilliSeconds(), res.GetMinStep()); UNIT_ASSERT_EQUAL(res.GetMaxStep(), std::numeric_limits::max()); - return {TPlanStep(res.GetMinStep())}; + return { TPlanStep(res.GetMinStep()) }; } return std::nullopt; } -} //namespace +} //namespace void ProposeSchemaTxFail(TTestBasicRuntime& runtime, TActorId& sender, const TString& txBody, const ui64 txId) { const auto result = ProposeSchemaTxOptional(runtime, sender, txBody, txId); @@ -142,8 +129,8 @@ ui32 WaitWriteResult(TTestBasicRuntime& runtime, ui64 shardId, std::vector return resWrite.GetStatus(); } -bool WriteDataImpl(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 tableId, const ui64 writeId, - const TString& data, const std::shared_ptr& schema, std::vector* writeIds, const NEvWrite::EModificationType mType, const ui64 lockId) { +bool WriteDataImpl(TTestBasicRuntime& runtime, TActorId& sender, const ui64 shardId, const ui64 tableId, const ui64 writeId, const TString& data, + const std::shared_ptr& schema, std::vector* writeIds, const NEvWrite::EModificationType mType, const ui64 lockId) { const TString dedupId = ToString(writeId); auto write = std::make_unique(writeId, NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE); @@ -182,8 +169,7 @@ bool WriteData(TTestBasicRuntime& runtime, TActorId& sender, const ui64 writeId, waitResult ? &ids : nullptr, mType, lockId); } -void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vector& pathIds, - NOlap::TSnapshot snap, ui64 scanId) { +void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vector& pathIds, NOlap::TSnapshot snap, ui64 scanId) { auto scan = std::make_unique(); auto& record = scan->Record; @@ -207,7 +193,7 @@ void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vec } for (ui64 pathId : pathIds) { - std::vector pk{TCell::Make(pathId)}; + std::vector pk{ TCell::Make(pathId) }; TSerializedTableRange range(TConstArrayRef(pk), true, TConstArrayRef(pk), true); auto newRange = record.MutableRanges()->Add(); range.Serialize(*newRange); @@ -220,8 +206,9 @@ void ScanIndexStats(TTestBasicRuntime& runtime, TActorId& sender, const std::vec ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, scan.release()); } -template -TPlanStep ProposeCommitCheck(TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& /* writeIds */, const ui64 lockId, Checker&& checker) { +template +TPlanStep ProposeCommitCheck(TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& /* writeIds */, + const ui64 lockId, Checker&& checker) { auto write = std::make_unique(txId, NKikimrDataEvents::TEvWrite::MODE_PREPARE); auto* lock = write->Record.MutableLocks()->AddLocks(); lock->SetLockId(lockId); @@ -237,7 +224,8 @@ TPlanStep ProposeCommitCheck(TTestBasicRuntime& runtime, TActorId& sender, ui64 return TPlanStep(res.GetMinStep()); } -TPlanStep ProposeCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& writeIds, const ui64 lockId) { +TPlanStep ProposeCommit( + TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& writeIds, const ui64 lockId) { const auto now = runtime.GetTimeProvider()->Now(); return ProposeCommitCheck(runtime, sender, shardId, txId, writeIds, lockId, [&](auto& res) { AFL_VERIFY(res.GetTxId() == txId)("tx_id", txId)("res", res.GetTxId()); @@ -248,7 +236,8 @@ TPlanStep ProposeCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 shard }); } -void ProposeCommitFail(TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& writeIds, const ui64 lockId) { +void ProposeCommitFail( + TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, ui64 txId, const std::vector& writeIds, const ui64 lockId) { const auto result = ProposeCommitCheck(runtime, sender, shardId, txId, writeIds, lockId, [&](auto& res) { UNIT_ASSERT_UNEQUAL(res.GetStatus(), NKikimrDataEvents::TEvWriteResult::STATUS_PREPARED); }); @@ -259,7 +248,6 @@ TPlanStep ProposeCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 txId, return ProposeCommit(runtime, sender, TTestTxConfig::TxTablet0, txId, writeIds, lockId); } - TPlanStep ProposeCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 txId, const std::vector& writeIds) { return ProposeCommit(runtime, sender, TTestTxConfig::TxTablet0, txId, writeIds); } @@ -298,10 +286,7 @@ void PlanCommit(TTestBasicRuntime& runtime, TActorId& sender, ui64 shardId, TPla TCell MakeTestCell(const TTypeInfo& typeInfo, ui32 value, std::vector& mem) { auto type = typeInfo.GetTypeId(); - if (type == NTypeIds::Utf8 || - type == NTypeIds::String || - type == NTypeIds::String4k || - type == NTypeIds::String2m) { + if (type == NTypeIds::Utf8 || type == NTypeIds::String || type == NTypeIds::String4k || type == NTypeIds::String2m) { mem.push_back(ToString(value)); const TString& str = mem.back(); return TCell(str.data(), str.size()); @@ -313,16 +298,14 @@ TCell MakeTestCell(const TTypeInfo& typeInfo, ui32 value, std::vector& mem.push_back("{ \"a\" = [ { \"b\" = 1; } ]; }"); const TString& str = mem.back(); return TCell(str.data(), str.size()); - } else if (type == NTypeIds::Timestamp || type == NTypeIds::Interval || - type == NTypeIds::Timestamp64 || type == NTypeIds::Interval64 || - type == NTypeIds::Uint64 || type == NTypeIds::Int64) { + } else if (type == NTypeIds::Timestamp || type == NTypeIds::Interval || type == NTypeIds::Timestamp64 || type == NTypeIds::Interval64 || + type == NTypeIds::Uint64 || type == NTypeIds::Int64) { return TCell::Make(value); } else if (type == NTypeIds::Uint32 || type == NTypeIds::Int32 || type == NTypeIds::Datetime) { return TCell::Make(value); } else if (type == NTypeIds::Uint16 || type == NTypeIds::Int16 || type == NTypeIds::Date) { return TCell::Make(value); - } else if (type == NTypeIds::Uint8 || type == NTypeIds::Int8 || type == NTypeIds::Byte || - type == NTypeIds::Bool) { + } else if (type == NTypeIds::Uint8 || type == NTypeIds::Int8 || type == NTypeIds::Byte || type == NTypeIds::Bool) { return TCell::Make(value); } else if (type == NTypeIds::Float) { return TCell::Make(value); @@ -345,9 +328,8 @@ std::vector MakeTestCells(const std::vector& types, ui32 value return cells; } - -TString MakeTestBlob(std::pair range, const std::vector& columns, - const TTestBlobOptions& options, const std::set& notNullColumns) { +TString MakeTestBlob(std::pair range, const std::vector& columns, const TTestBlobOptions& options, + const std::set& notNullColumns) { NArrow::TArrowBatchBuilder batchBuilder(arrow::Compression::LZ4_FRAME, notNullColumns); const auto startStatus = batchBuilder.Start(NArrow::NTest::TTestColumn::ConvertToPairs(columns)); UNIT_ASSERT_C(startStatus.ok(), startStatus.ToString()); @@ -397,15 +379,14 @@ TString MakeTestBlob(std::pair range, const std::vector range, bool inclusiveFrom, bool inclusiveTo, - const std::vector& columns) { +TSerializedTableRange MakeTestRange( + std::pair range, bool inclusiveFrom, bool inclusiveTo, const std::vector& columns) { std::vector mem; std::vector types = TTestSchema::ExtractTypes(columns); std::vector cellsFrom = MakeTestCells(types, range.first, mem); std::vector cellsTo = MakeTestCells(types, range.second, mem); - return TSerializedTableRange(TConstArrayRef(cellsFrom), inclusiveFrom, - TConstArrayRef(cellsTo), inclusiveTo); + return TSerializedTableRange(TConstArrayRef(cellsFrom), inclusiveFrom, TConstArrayRef(cellsTo), inclusiveTo); } THashMap TTestSchema::BuildSnapshot(const TTableSpecials& specials) { @@ -431,7 +412,6 @@ THashMap TTestSchema::BuildSnapshot( void TTestSchema::InitSchema(const std::vector& columns, const std::vector& pk, const TTableSpecials& specials, NKikimrSchemeOp::TColumnTableSchema* schema) { - for (ui32 i = 0; i < columns.size(); ++i) { *schema->MutableColumns()->Add() = columns[i].CreateColumn(i + 1); if (!specials.NeedTestStatistics(pk)) { @@ -440,7 +420,7 @@ void TTestSchema::InitSchema(const std::vector& colu if (NOlap::NIndexes::NMax::TIndexMeta::IsAvailableType(columns[i].GetType())) { *schema->AddIndexes() = NOlap::NIndexes::TIndexMetaContainer( std::make_shared(1000 + i, "MAX::INDEX::" + columns[i].GetName(), "__LOCAL_METADATA", i + 1)) - .SerializeToProto(); + .SerializeToProto(); } } @@ -458,107 +438,109 @@ void TTestSchema::InitSchema(const std::vector& colu if (specials.GetUseForcedCompaction()) { NKikimrSchemeOp::TCompactionPlannerConstructorContainer::TLOptimizer optimizer; *schema->MutableOptions()->MutableCompactionPlannerConstructor()->MutableLBuckets() = optimizer; - schema->MutableOptions()->MutableCompactionPlannerConstructor()->SetClassName("l-buckets"); //TODO use appropriate lc-buckets configuration + schema->MutableOptions()->MutableCompactionPlannerConstructor()->SetClassName( + "l-buckets"); //TODO use appropriate lc-buckets configuration } } -} +} // namespace NKikimr::NTxUT namespace NKikimr::NColumnShard { - NOlap::TIndexInfo BuildTableInfo(const std::vector& ydbSchema, - const std::vector& key) { - THashMap columns; - THashMap columnIdByName; - for (ui32 i = 0; i < ydbSchema.size(); ++i) { - ui32 id = i + 1; - auto& name = ydbSchema[i].GetName(); - auto& type = ydbSchema[i].GetType(); - - columns[id] = NTable::TColumn(name, id, type, ""); - AFL_VERIFY(columnIdByName.emplace(name, id).second); - } - - std::vector pkIds; - ui32 idx = 0; - for (const auto& c : key) { - auto it = columnIdByName.FindPtr(c.GetName()); - AFL_VERIFY(it); - AFL_VERIFY(*it < columns.size()); - columns[*it].KeyOrder = idx++; - pkIds.push_back(*it); - } - return NOlap::TIndexInfo::BuildDefault(NOlap::TTestStoragesManager::GetInstance(), columns, pkIds); +NOlap::TIndexInfo BuildTableInfo(const std::vector& ydbSchema, const std::vector& key) { + THashMap columns; + THashMap columnIdByName; + for (ui32 i = 0; i < ydbSchema.size(); ++i) { + ui32 id = i + 1; + auto& name = ydbSchema[i].GetName(); + auto& type = ydbSchema[i].GetType(); + + columns[id] = NTable::TColumn(name, id, type, ""); + AFL_VERIFY(columnIdByName.emplace(name, id).second); } - NTxUT::TPlanStep SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, const TString& txBody, const ui64 txId) { - - auto controller = NYDBTest::TControllers::GetControllerAs(); - while (controller && !controller->IsActiveTablet(TTestTxConfig::TxTablet0)) { - runtime.SimulateSleep(TDuration::Seconds(1)); - } - - using namespace NTxUT; - const auto planStep = ProposeSchemaTx(runtime, sender, txBody, txId); - PlanSchemaTx(runtime, sender, NOlap::TSnapshot(planStep, txId)); - return planStep; + std::vector pkIds; + ui32 idx = 0; + for (const auto& c : key) { + auto it = columnIdByName.FindPtr(c.GetName()); + AFL_VERIFY(it); + AFL_VERIFY(*it < columns.size()); + columns[*it].KeyOrder = idx++; + pkIds.push_back(*it); } + return NOlap::TIndexInfo::BuildDefault(NOlap::TTestStoragesManager::GetInstance(), columns, pkIds); +} - NTxUT::TPlanStep SetupSchema( - TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, const TestTableDescription& table, TString codec, const ui64 txId) { - using namespace NTxUT; - TString txBody; - auto specials = TTestSchema::TTableSpecials().WithCodec(codec); - if (table.InStore) { - txBody = TTestSchema::CreateInitShardTxBody(pathId, table.Schema, table.Pk, specials); - } else { - txBody = TTestSchema::CreateStandaloneTableTxBody(pathId, table.Schema, table.Pk, specials); - } - return SetupSchema(runtime, sender, txBody, txId); +NTxUT::TPlanStep SetupSchema(TTestBasicRuntime& runtime, TActorId& sender, const TString& txBody, const ui64 txId) { + auto controller = NYDBTest::TControllers::GetControllerAs(); + while (controller && !controller->IsActiveTablet(TTestTxConfig::TxTablet0)) { + runtime.SimulateSleep(TDuration::Seconds(1)); } - NTxUT::TPlanStep PrepareTablet(TTestBasicRuntime& runtime, const ui64 tableId, const std::vector& schema, const ui32 keySize) { - using namespace NTxUT; - CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); + using namespace NTxUT; + const auto planStep = ProposeSchemaTx(runtime, sender, txBody, txId); + PlanSchemaTx(runtime, sender, NOlap::TSnapshot(planStep, txId)); + return planStep; +} - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); - runtime.DispatchEvents(options); +NTxUT::TPlanStep SetupSchema( + TTestBasicRuntime& runtime, TActorId& sender, ui64 pathId, const TestTableDescription& table, TString codec, const ui64 txId) { + using namespace NTxUT; + TString txBody; + auto specials = TTestSchema::TTableSpecials().WithCodec(codec); + if (table.InStore) { + txBody = TTestSchema::CreateInitShardTxBody(pathId, table.Schema, table.Pk, specials); + } else { + txBody = TTestSchema::CreateStandaloneTableTxBody(pathId, table.Schema, table.Pk, specials); + } + return SetupSchema(runtime, sender, txBody, txId); +} - TestTableDescription tableDescription; - tableDescription.Schema = schema; - tableDescription.Pk = {}; - for (ui64 i = 0; i < keySize; ++i) { - Y_ABORT_UNLESS(i < schema.size()); - tableDescription.Pk.push_back(schema[i]); - } - TActorId sender = runtime.AllocateEdgeActor(); - return SetupSchema(runtime, sender, tableId, tableDescription); +NTxUT::TPlanStep PrepareTablet( + TTestBasicRuntime& runtime, const ui64 tableId, const std::vector& schema, const ui32 keySize) { + using namespace NTxUT; + CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); + + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); + runtime.DispatchEvents(options); + + TestTableDescription tableDescription; + tableDescription.Schema = schema; + tableDescription.Pk = {}; + for (ui64 i = 0; i < keySize; ++i) { + Y_ABORT_UNLESS(i < schema.size()); + tableDescription.Pk.push_back(schema[i]); } + TActorId sender = runtime.AllocateEdgeActor(); + return SetupSchema(runtime, sender, tableId, tableDescription); +} - NTxUT::TPlanStep PrepareTablet(TTestBasicRuntime& runtime, const TString& schemaTxBody) { - using namespace NTxUT; - CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); +NTxUT::TPlanStep PrepareTablet(TTestBasicRuntime& runtime, const TString& schemaTxBody) { + using namespace NTxUT; + CreateTestBootstrapper(runtime, CreateTestTabletInfo(TTestTxConfig::TxTablet0, TTabletTypes::ColumnShard), &CreateColumnShard); - TDispatchOptions options; - options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); - runtime.DispatchEvents(options); + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); + runtime.DispatchEvents(options); - TActorId sender = runtime.AllocateEdgeActor(); - return SetupSchema(runtime, sender, schemaTxBody, 100); + TActorId sender = runtime.AllocateEdgeActor(); + return SetupSchema(runtime, sender, schemaTxBody, 100); +} + +std::shared_ptr ReadAllAsBatch( + TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector& schema) { + std::vector fields; + ui32 idx = 1; + for (auto&& f : schema) { + Y_UNUSED(f); + fields.emplace_back(idx++); } - std::shared_ptr ReadAllAsBatch(TTestBasicRuntime& runtime, const ui64 tableId, const NOlap::TSnapshot& snapshot, const std::vector& schema) { - std::vector fields; - ui32 idx = 1; - for (auto&& f : schema) { - Y_UNUSED(f); - fields.emplace_back(idx++); - } - - NTxUT::TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, snapshot); - reader.SetReplyColumnIds(fields); - auto rb = reader.ReadAll(); - //UNIT_ASSERT(reader.IsCorrectlyFinished()); - return rb ? rb : NArrow::MakeEmptyBatch(NArrow::MakeArrowSchema(schema)); - } + NTxUT::TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, snapshot); + reader.SetReplyColumnIds(fields); + auto rb = reader.ReadAll(); + //UNIT_ASSERT(reader.IsCorrectlyFinished()); + return rb ? rb : NArrow::MakeEmptyBatch(NArrow::MakeArrowSchema(schema)); } + +} // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h index f3ad161fe45a..38f878b400ed 100644 --- a/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h +++ b/ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h @@ -26,8 +26,6 @@ struct TIndexInfo; namespace NKikimr::NTxUT { -void SetupSharedMetadataAccessorCacheService(TTestActorRuntime& runtime, ui32 nodeIndex); - using TPlanStep = TPositiveIncreasingControlInteger; // Private events of different actors reuse the same ES_PRIVATE range diff --git a/ydb/core/tx/columnshard/ya.make b/ydb/core/tx/columnshard/ya.make index e2eaf6279c25..e7238e4007ad 100644 --- a/ydb/core/tx/columnshard/ya.make +++ b/ydb/core/tx/columnshard/ya.make @@ -64,6 +64,7 @@ PEERDIR( ydb/core/tx/columnshard/tx_reader ydb/core/tx/conveyor/usage ydb/core/tx/conveyor_composite/service + ydb/core/tx/general_cache/usage ydb/core/tx/long_tx_service/public ydb/core/tx/priorities/service ydb/core/tx/tiering diff --git a/ydb/core/tx/conveyor_composite/service/common.cpp b/ydb/core/tx/conveyor_composite/service/common.cpp index bc1e111dfca5..163b58945037 100644 --- a/ydb/core/tx/conveyor_composite/service/common.cpp +++ b/ydb/core/tx/conveyor_composite/service/common.cpp @@ -5,13 +5,7 @@ namespace NKikimr::NConveyorComposite { void TCPUUsage::Exchange(const TDuration predicted, const TMonotonic start, const TMonotonic finish) { - { - TTaskCPUUsage usage(start, finish); - if (usage.Cut(StartInstant)) { - Usage.emplace_back(TTaskCPUUsage(start, finish)); - Duration += Usage.back().GetDuration(); - } - } + Duration += finish - start; AFL_VERIFY(predicted <= PredictedDuration)("predicted_delta", predicted)("predicted_sum", PredictedDuration); PredictedDuration -= predicted; if (Parent) { @@ -19,21 +13,4 @@ void TCPUUsage::Exchange(const TDuration predicted, const TMonotonic start, cons } } -void TCPUUsage::Cut(const TMonotonic start) { - StartInstant = start; - ui32 idx = 0; - while (idx < Usage.size()) { - AFL_VERIFY(Usage[idx].GetDuration() <= Duration); - Duration -= Usage[idx].GetDuration(); - if (Usage[idx].GetFinish() <= start) { - std::swap(Usage[idx], Usage.back()); - Usage.pop_back(); - } else { - AFL_VERIFY(Usage[idx].Cut(start)); - Duration += Usage[idx].GetDuration(); - ++idx; - } - } -} - } // namespace NKikimr::NConveyorComposite diff --git a/ydb/core/tx/conveyor_composite/service/common.h b/ydb/core/tx/conveyor_composite/service/common.h index df42187b86ad..051a52e70f62 100644 --- a/ydb/core/tx/conveyor_composite/service/common.h +++ b/ydb/core/tx/conveyor_composite/service/common.h @@ -60,7 +60,6 @@ class TAverageCalcer { class TCPUUsage { private: - std::deque Usage; YDB_READONLY_DEF(TDuration, Duration); YDB_READONLY_DEF(TDuration, PredictedDuration); std::shared_ptr Parent; @@ -72,7 +71,6 @@ class TCPUUsage { } void Clear() { - Usage.clear(); Duration = TDuration::Zero(); PredictedDuration = TDuration::Zero(); StartInstant = TMonotonic::Zero(); @@ -93,19 +91,14 @@ class TCPUUsage { } } - void AddUsage(const TTaskCPUUsage& extUsage) { - TTaskCPUUsage usage = extUsage; - if (usage.Cut(StartInstant)) { - Usage.emplace_back(usage); - Duration += usage.GetDuration(); - } + void AddUsage(const TTaskCPUUsage& usage) { + Duration += usage.GetDuration(); if (Parent) { - Parent->AddUsage(extUsage); + Parent->AddUsage(usage); } } void Exchange(const TDuration predicted, const TMonotonic start, const TMonotonic finish); - void Cut(const TMonotonic start); }; class TCPUGroup { diff --git a/ydb/core/tx/general_cache/service/counters.cpp b/ydb/core/tx/general_cache/service/counters.cpp new file mode 100644 index 000000000000..3388aef1511c --- /dev/null +++ b/ydb/core/tx/general_cache/service/counters.cpp @@ -0,0 +1,5 @@ +#include "counters.h" + +namespace NKikimr::NGeneralCache::NPrivate { + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/counters.h b/ydb/core/tx/general_cache/service/counters.h new file mode 100644 index 000000000000..d42d1fa35b94 --- /dev/null +++ b/ydb/core/tx/general_cache/service/counters.h @@ -0,0 +1,70 @@ +#pragma once +#include + +namespace NKikimr::NGeneralCache::NPrivate { + +class TManagerCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + const NMonitoring::THistogramPtr RequestDuration; + +public: + void OnRequestFinished(const TDuration d) const { + RequestDuration->Collect(d.MicroSeconds()); + } + + const NMonitoring::TDynamicCounters::TCounterPtr RequestCacheMiss; + const NMonitoring::TDynamicCounters::TCounterPtr RequestCacheHit; + const NMonitoring::TDynamicCounters::TCounterPtr ObjectCacheMiss; + const NMonitoring::TDynamicCounters::TCounterPtr ObjectCacheHit; + const NMonitoring::TDynamicCounters::TCounterPtr CacheSizeCount; + const NMonitoring::TDynamicCounters::TCounterPtr CacheSizeBytes; + const NMonitoring::TDynamicCounters::TCounterPtr ObjectsInFlight; + const NMonitoring::TDynamicCounters::TCounterPtr RequestsInFlight; + const NMonitoring::TDynamicCounters::TCounterPtr RequestsQueueSize; + const NMonitoring::TDynamicCounters::TCounterPtr ObjectsQueueSize; + const NMonitoring::TDynamicCounters::TCounterPtr IncomingRequestsCount; + const NMonitoring::TDynamicCounters::TCounterPtr DirectRequests; + const NMonitoring::TDynamicCounters::TCounterPtr AdditionalObjectInfo; + const NMonitoring::TDynamicCounters::TCounterPtr RemovedObjectInfo; + const NMonitoring::TDynamicCounters::TCounterPtr FetchedObject; + const NMonitoring::TDynamicCounters::TCounterPtr NoExistsObject; + const NMonitoring::TDynamicCounters::TCounterPtr FailedObject; + + TManagerCounters(NColumnShard::TCommonCountersOwner& base) + : TBase(base, "signals_owner", "manager") + , RequestDuration(TBase::GetHistogram("Requests/Duration/Us", NMonitoring::ExponentialHistogram(15, 2, 16))) + , RequestCacheMiss(TBase::GetDeriviative("Cache/Request/Miss/Count")) + , RequestCacheHit(TBase::GetDeriviative("Cache/Request/Hit/Count")) + , ObjectCacheMiss(TBase::GetDeriviative("Cache/Object/Miss/Count")) + , ObjectCacheHit(TBase::GetDeriviative("Cache/Object/Hit/Count")) + , CacheSizeCount(TBase::GetValue("Cache/Size/Count")) + , CacheSizeBytes(TBase::GetValue("Cache/Size/Bytes")) + , ObjectsInFlight(TBase::GetValue("DirectRequest/ObjectsInFlight/Count")) + , RequestsInFlight(TBase::GetValue("DirectRequest/RequestsInFlight/Count")) + , RequestsQueueSize(TBase::GetValue("RequestsQueue/Size/Count")) + , ObjectsQueueSize(TBase::GetValue("ObjectsQueue/Size/Count")) + , IncomingRequestsCount(TBase::GetDeriviative("Incoming/Requests/Count")) + , DirectRequests(TBase::GetDeriviative("DirectRequest/Count")) + , AdditionalObjectInfo(TBase::GetDeriviative("AdditionalInfo/Count")) + , RemovedObjectInfo(TBase::GetDeriviative("RemovedInfo/Count")) + , FetchedObject(TBase::GetDeriviative("DirectRequest/Fetched/Count")) + , NoExistsObject(TBase::GetDeriviative("DirectRequest/NoExists/Count")) + , FailedObject(TBase::GetDeriviative("DirectRequest/Failed/Count")) { + } +}; + +class TActorCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + YDB_READONLY_DEF(std::shared_ptr, Manager); + +public: + TActorCounters(const TString& cacheName, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& baseCounters) + : TBase("general_cache", baseCounters) { + DeepSubGroup("cache_name", cacheName); + Manager = std::make_shared(*this); + } +}; + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/manager.cpp b/ydb/core/tx/general_cache/service/manager.cpp new file mode 100644 index 000000000000..923274722412 --- /dev/null +++ b/ydb/core/tx/general_cache/service/manager.cpp @@ -0,0 +1,5 @@ +#include "manager.h" + +namespace NKikimr::NGeneralCache::NPrivate { + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/manager.h b/ydb/core/tx/general_cache/service/manager.h new file mode 100644 index 000000000000..fade980bd1c5 --- /dev/null +++ b/ydb/core/tx/general_cache/service/manager.h @@ -0,0 +1,226 @@ +#pragma once +#include "counters.h" + +#include +#include +#include + +#include +#include + +#include + +namespace NKikimr::NGeneralCache::NPrivate { + +template +class TRequest: public NColumnShard::TMonitoringObjectsCounter>, public TNonCopyable { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + using ICallback = NPublic::ICallback; + static inline TAtomicCounter Counter = 0; + YDB_READONLY(ui64, RequestId, Counter.Inc()); + YDB_READONLY(TMonotonic, Created, TMonotonic::Now()); + YDB_READONLY_DEF(THashSet, Wait); + THashMap Result; + THashSet Removed; + THashMap Errors; + + std::shared_ptr Callback; + const EConsumer Consumer; + +public: + EConsumer GetConsumer() const { + return Consumer; + } + + [[nodiscard]] bool AddResult(const TAddress& addr, const TObject& obj) { + AFL_VERIFY(Wait.erase(addr)); + AFL_VERIFY(Result.emplace(addr, obj).second); + if (Wait.empty()) { + Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); + } + return Wait.empty(); + } + + [[nodiscard]] bool AddRemoved(const TAddress& addr) { + AFL_VERIFY(Wait.erase(addr)); + AFL_VERIFY(Removed.emplace(addr).second); + if (Wait.empty()) { + Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); + } + return Wait.empty(); + } + + [[nodiscard]] bool AddError(const TAddress& addr, const TString& errorMessage) { + AFL_VERIFY(Wait.erase(addr)); + AFL_VERIFY(Errors.emplace(addr, errorMessage).second); + if (Wait.empty()) { + Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); + } + return Wait.empty(); + } + + TRequest(THashSet&& addresses, std::shared_ptr&& callback, const EConsumer consumer) + : Wait(addresses) + , Callback(callback) + , Consumer(consumer) { + } +}; + +template +class TManager { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + using TRequest = TRequest; + + const NPublic::TConfig Config; + const TString CacheName = TPolicy::GetCacheName(); + const std::shared_ptr Counters; + std::shared_ptr> ObjectsProcessor; + TLRUCache Cache; + THashMap>> RequestedObjects; + THashSet RequestsInProgress; + std::deque> RequestsQueue; + TPositiveControlInteger QueueObjectsCount; + + void DrainQueue() { + THashMap> requestedAddresses; + while (RequestsQueue.size() && RequestedObjects.size() < Config.GetDirectInflightLimit()) { + auto request = std::move(RequestsQueue.front()); + RequestsQueue.pop_front(); + QueueObjectsCount.Sub(request->GetWait().size()); + AFL_VERIFY(RequestsInProgress.emplace(request->GetRequestId()).second); + auto& addresses = requestedAddresses[request->GetConsumer()]; + for (auto&& i : request->GetWait()) { + auto it = RequestedObjects.find(i); + if (it == RequestedObjects.end()) { + it = RequestedObjects.emplace(i, std::vector>()).first; + AFL_VERIFY(addresses.emplace(i).second); + } + it->second.emplace_back(request); + } + } + ObjectsProcessor->AskData(std::move(requestedAddresses), ObjectsProcessor); + Counters->DirectRequests->Inc(); + Counters->RequestsQueueSize->Set(RequestsQueue.size()); + Counters->ObjectsQueueSize->Set(QueueObjectsCount.Val()); + Counters->ObjectsInFlight->Set(RequestedObjects.size()); + Counters->RequestsInFlight->Set(RequestsInProgress.size()); + Counters->CacheSizeCount->Set(Cache.Size()); + Counters->CacheSizeBytes->Set(Cache.TotalSize()); + } + + void AddObjects(THashMap&& add, const bool isAdditional, const TMonotonic now) { + for (auto&& i : add) { + auto it = RequestedObjects.find(i.first); + Cache.Insert(i.first, i.second); + if (it != RequestedObjects.end()) { + for (auto&& r : it->second) { + if (r->AddResult(i.first, i.second)) { + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + if (isAdditional) { + Counters->AdditionalObjectInfo->Inc(); + } else { + Counters->FetchedObject->Inc(); + } + } + + } + RequestedObjects.erase(it); + } + } + } + + void RemoveObjects(THashSet&& remove, const bool isAdditional, const TMonotonic now) { + for (auto&& i : remove) { + auto it = RequestedObjects.find(i); + if (it != RequestedObjects.end()) { + for (auto&& r : it->second) { + if (r->AddRemoved(i)) { + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + if (isAdditional) { + Counters->RemovedObjectInfo->Inc(); + } else { + Counters->NoExistsObject->Inc(); + } + } + } + RequestedObjects.erase(it); + } + } + } + +public: + TManager(const NPublic::TConfig& config, const NActors::TActorId& ownerActorId, + const std::shared_ptr& counters) + : Config(config) + , Counters(counters) + , ObjectsProcessor(TPolicy::BuildObjectsProcessor(ownerActorId)) + , Cache(Config.GetMemoryLimit()) { + AFL_NOTICE(NKikimrServices::GENERAL_CACHE)("event", "general_cache_manager")("owner_actor_id", ownerActorId)("config", config.DebugString()); + } + + void AddRequest(const std::shared_ptr& request) { + AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "add_request"); + THashMap objectsResult; + Counters->IncomingRequestsCount->Inc(); + for (auto&& i : request->GetWait()) { + auto it = Cache.Find(i); + if (it == Cache.End()) { + Counters->ObjectCacheMiss->Inc(); + } else { + Counters->ObjectCacheHit->Inc(); + AFL_VERIFY(objectsResult.emplace(i, it.Value()).second); + } + } + for (auto&& i : objectsResult) { + Y_UNUSED(request->AddResult(i.first, std::move(i.second))); + } + if (request->GetWait().empty()) { + Counters->RequestCacheHit->Inc(); + return; + } else { + Counters->RequestCacheMiss->Inc(); + } + QueueObjectsCount.Add(request->GetWait().size()); + RequestsQueue.emplace_back(request); + DrainQueue(); + } + + void OnAdditionalObjectsInfo(THashMap&& add, THashSet&& remove) { + AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "objects_info"); + const TMonotonic now = TMonotonic::Now(); + AddObjects(std::move(add), true, now); + RemoveObjects(std::move(remove), true, now); + DrainQueue(); + } + + void OnRequestResult(THashMap&& objects, THashSet&& removed, THashMap&& failed) { + AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "on_result"); + const TMonotonic now = TMonotonic::Now(); + AddObjects(std::move(objects), false, now); + RemoveObjects(std::move(removed), false, now); + + for (auto&& i : failed) { + auto it = RequestedObjects.find(i.first); + AFL_VERIFY(it != RequestedObjects.end()); + for (auto&& r : it->second) { + if (r->AddError(i.first, i.second)) { + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + Counters->FailedObject->Inc(); + } + } + RequestedObjects.erase(it); + } + DrainQueue(); + } +}; + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/service.cpp b/ydb/core/tx/general_cache/service/service.cpp new file mode 100644 index 000000000000..f7fed73edd66 --- /dev/null +++ b/ydb/core/tx/general_cache/service/service.cpp @@ -0,0 +1,5 @@ +#include "service.h" + +namespace NKikimr::NGeneralCache::NPrivate { + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/service.h b/ydb/core/tx/general_cache/service/service.h new file mode 100644 index 000000000000..f95f120bdeb0 --- /dev/null +++ b/ydb/core/tx/general_cache/service/service.h @@ -0,0 +1,64 @@ +#pragma once +#include "counters.h" +#include "manager.h" + +#include +#include +#include + +#include + +namespace NKikimr::NGeneralCache::NPrivate { + +template +class TDistributor: public TActorBootstrapped> { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using TManager = TManager; + using TRequest = TRequest; + + using TBase = TActorBootstrapped>; + const NPublic::TConfig Config; + const TActorCounters Counters; + std::unique_ptr Manager; + + void HandleMain(NPublic::TEvents::TEvAskData::TPtr& ev) { + Manager->AddRequest(std::make_shared(ev->Get()->ExtractAddresses(), ev->Get()->ExtractCallback(), ev->Get()->GetConsumer())); + } + void HandleMain(NSource::TEvents::TEvObjectsInfo::TPtr& ev) { + Manager->OnRequestResult(ev->Get()->ExtractObjects(), ev->Get()->ExtractRemoved(), ev->Get()->ExtractErrors()); + } + void HandleMain(NSource::TEvents::TEvAdditionalObjectsInfo::TPtr& ev) { + Manager->OnAdditionalObjectsInfo(ev->Get()->ExtractAddObjects(), ev->Get()->ExtractRemoveObjects()); + } + +public: + STATEFN(StateMain) { + // NActors::TLogContextGuard lGuard = NActors::TLogContextBuilder::Build()("name", ConveyorName) + // ("workers", Workers.size())("waiting", Waiting.size())("actor_id", SelfId()); + switch (ev->GetTypeRewrite()) { + hFunc(NPublic::TEvents::TEvAskData, HandleMain); + hFunc(NSource::TEvents::TEvObjectsInfo, HandleMain); + hFunc(NSource::TEvents::TEvAdditionalObjectsInfo, HandleMain); + default: + AFL_ERROR(NKikimrServices::TX_CONVEYOR)("problem", "unexpected event for general cache")("ev_type", ev->GetTypeName()); + break; + } + } + + TDistributor(const NPublic::TConfig& config, const TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) + : Config(config) + , Counters(TPolicy::GetCacheName(), conveyorSignals) { + } + + ~TDistributor() { + } + + void Bootstrap() { + Manager = std::make_unique(Config, TBase::SelfId(), Counters.GetManager()); + TBase::Become(&TDistributor::StateMain); + } +}; + +} // namespace NKikimr::NGeneralCache::NPrivate diff --git a/ydb/core/tx/general_cache/service/ya.make b/ydb/core/tx/general_cache/service/ya.make new file mode 100644 index 000000000000..17537a3bc3f9 --- /dev/null +++ b/ydb/core/tx/general_cache/service/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + manager.cpp + counters.cpp + service.cpp +) + +PEERDIR( + ydb/core/protos +) + +END() diff --git a/ydb/core/tx/general_cache/source/abstract.cpp b/ydb/core/tx/general_cache/source/abstract.cpp new file mode 100644 index 000000000000..ff004fcc6fae --- /dev/null +++ b/ydb/core/tx/general_cache/source/abstract.cpp @@ -0,0 +1,5 @@ +#include "abstract.h" + +namespace NKikimr::NGeneralCache::NSource { + +} // namespace NKikimr::NGeneralCache::NSource diff --git a/ydb/core/tx/general_cache/source/abstract.h b/ydb/core/tx/general_cache/source/abstract.h new file mode 100644 index 000000000000..3366b073735c --- /dev/null +++ b/ydb/core/tx/general_cache/source/abstract.h @@ -0,0 +1,35 @@ +#pragma once +#include + +#include +#include + +namespace NKikimr::NGeneralCache::NSource { + +template +class IObjectsProcessor { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + + using TSelf = IObjectsProcessor; + virtual void DoAskData( + const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const = 0; + virtual void DoOnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + THashMap&& errorAddresses) const = 0; + +public: + virtual ~IObjectsProcessor() = default; + + void AskData(const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const { + DoAskData(objectAddressesByConsumer, selfPtr); + } + + void OnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + THashMap&& errorAddresses) const { + DoOnReceiveData(std::move(objectAddresses), std::move(removedAddresses), std::move(errorAddresses)); + } +}; + +} // namespace NKikimr::NGeneralCache::NSource diff --git a/ydb/core/tx/general_cache/source/events.cpp b/ydb/core/tx/general_cache/source/events.cpp new file mode 100644 index 000000000000..39bec14e18a0 --- /dev/null +++ b/ydb/core/tx/general_cache/source/events.cpp @@ -0,0 +1,5 @@ +#include "events.h" + +namespace NKikimr::NGeneralCache::NSource { + +} // namespace NKikimr::NGeneralCache::NSource diff --git a/ydb/core/tx/general_cache/source/events.h b/ydb/core/tx/general_cache/source/events.h new file mode 100644 index 000000000000..d515dbe3a163 --- /dev/null +++ b/ydb/core/tx/general_cache/source/events.h @@ -0,0 +1,91 @@ +#pragma once +#include + +#include +#include + +#include +#include + +namespace NKikimr::NGeneralCache::NSource { + +template +struct TEvents { + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + + enum EEv { + EvObjectsInfo = EventSpaceBegin(TKikimrEvents::ES_GENERAL_CACHE_SOURCE), + EvAdditionalObjectsInfo, + EvEnd + }; + + class TEvObjectsInfo: public NActors::TEventLocal { + private: + bool RemovedExtracted = false; + THashSet Removed; + + bool ObjectsExtracted = false; + THashMap Objects; + + bool ErrorsExtracted = false; + THashMap Errors; + + public: + THashMap ExtractObjects() { + AFL_VERIFY(!ObjectsExtracted); + ObjectsExtracted = true; + return std::move(Objects); + } + + THashSet ExtractRemoved() { + AFL_VERIFY(!RemovedExtracted); + RemovedExtracted = true; + return std::move(Removed); + } + + THashMap ExtractErrors() { + AFL_VERIFY(!ErrorsExtracted); + ErrorsExtracted = true; + return std::move(Errors); + } + + TEvObjectsInfo(THashMap&& objects, THashSet&& removed, THashMap&& errors) + : Removed(std::move(removed)) + , Objects(std::move(objects)) + , Errors(std::move(errors)) + { + } + }; + + class TEvAdditionalObjectsInfo: public NActors::TEventLocal { + private: + bool AddObjectsExtracted = false; + THashMap AddObjects; + + bool RemoveObjectsExtracted = false; + THashSet RemoveObjects; + + public: + THashMap ExtractAddObjects() { + AFL_VERIFY(!AddObjectsExtracted); + AddObjectsExtracted = true; + return std::move(AddObjects); + } + + THashSet ExtractRemoveObjects() { + AFL_VERIFY(!RemoveObjectsExtracted); + RemoveObjectsExtracted = true; + return std::move(RemoveObjects); + } + + TEvAdditionalObjectsInfo(THashMap&& add, THashSet&& remove) + : AddObjects(std::move(add)) + , RemoveObjects(std::move(remove)) + { + } + }; +}; + +} // namespace NKikimr::NGeneralCache::NSource diff --git a/ydb/core/tx/general_cache/source/ya.make b/ydb/core/tx/general_cache/source/ya.make new file mode 100644 index 000000000000..f7a749809bac --- /dev/null +++ b/ydb/core/tx/general_cache/source/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + abstract.cpp + events.cpp +) + +PEERDIR( + ydb/library/actors/core + ydb/core/protos +) + +END() diff --git a/ydb/core/tx/general_cache/usage/abstract.cpp b/ydb/core/tx/general_cache/usage/abstract.cpp new file mode 100644 index 000000000000..ff004fcc6fae --- /dev/null +++ b/ydb/core/tx/general_cache/usage/abstract.cpp @@ -0,0 +1,5 @@ +#include "abstract.h" + +namespace NKikimr::NGeneralCache::NSource { + +} // namespace NKikimr::NGeneralCache::NSource diff --git a/ydb/core/tx/general_cache/usage/abstract.h b/ydb/core/tx/general_cache/usage/abstract.h new file mode 100644 index 000000000000..2766d0ffb72f --- /dev/null +++ b/ydb/core/tx/general_cache/usage/abstract.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include + +namespace NKikimr::NGeneralCache::NPublic { + +template +class ICallback { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + + virtual void DoOnResultReady(THashMap&& objectAddresses, THashSet&& removedAddresses, + THashMap&& errorAddresses) const = 0; + +public: + virtual ~ICallback() = default; + + void OnResultReady(THashMap&& objectAddresses, THashSet&& removedAddresses, + THashMap&& errorAddresses) const { + DoOnResultReady(std::move(objectAddresses), std::move(removedAddresses), std::move(errorAddresses)); + } +}; + +} // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/config.cpp b/ydb/core/tx/general_cache/usage/config.cpp new file mode 100644 index 000000000000..097f9c3ce768 --- /dev/null +++ b/ydb/core/tx/general_cache/usage/config.cpp @@ -0,0 +1,35 @@ +#include "config.h" + +#include + +namespace NKikimr::NGeneralCache::NPublic { + +TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TGeneralCacheConfig& config) { + if (config.HasDirectInflightLimit()) { + DirectInflightLimit = config.GetDirectInflightLimit(); + } + if (config.HasMemoryLimit()) { + MemoryLimit = config.GetMemoryLimit(); + } + return TConclusionStatus::Success(); +} + +TConclusion TConfig::BuildFromProto(const NKikimrConfig::TGeneralCacheConfig& protoConfig) { + TConfig config; + auto conclusion = config.DeserializeFromProto(protoConfig); + if (conclusion.IsFail()) { + return conclusion; + } + return config; +} + +TString TConfig::DebugString() const { + TStringBuilder sb; + sb << "{"; + sb << "MemoryLimit=" << MemoryLimit << ";"; + sb << "DirectInflightLimit=" << DirectInflightLimit << ";"; + sb << "}"; + return sb; +} + +} // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/config.h b/ydb/core/tx/general_cache/usage/config.h new file mode 100644 index 000000000000..bac2543131fb --- /dev/null +++ b/ydb/core/tx/general_cache/usage/config.h @@ -0,0 +1,28 @@ +#pragma once +#include + +#include +#include + +namespace NKikimr::NGeneralCache::NPublic { + +class TConfig { +private: + YDB_READONLY(ui32, MemoryLimit, ((ui64)1 << 30)); + YDB_READONLY(ui32, DirectInflightLimit, 1000); + + TConfig() = default; + [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TGeneralCacheConfig& config); + +public: + static TConfig BuildDefault() { + static TConfig result = TConfig(); + return result; + } + + static TConclusion BuildFromProto(const NKikimrConfig::TGeneralCacheConfig& protoConfig); + + TString DebugString() const; +}; + +} // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/events.cpp b/ydb/core/tx/general_cache/usage/events.cpp new file mode 100644 index 000000000000..9f4d3faf2aae --- /dev/null +++ b/ydb/core/tx/general_cache/usage/events.cpp @@ -0,0 +1,5 @@ +#include "events.h" + +namespace NKikimr::NGeneralCache::NPublic { + +} // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/events.h b/ydb/core/tx/general_cache/usage/events.h new file mode 100644 index 000000000000..4f4994c316c7 --- /dev/null +++ b/ydb/core/tx/general_cache/usage/events.h @@ -0,0 +1,56 @@ +#pragma once +#include "abstract.h" + +#include +#include + +#include +#include +#include + +namespace NKikimr::NGeneralCache::NPublic { + +template +struct TEvents { + using TAddress = typename TPolicy::TAddress; + using EConsumer = typename TPolicy::EConsumer; + using ICallback = ICallback; + + enum EEv { + EvAskData = EventSpaceBegin(TKikimrEvents::ES_GENERAL_CACHE_PUBLIC), + EvEnd + }; + + static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_GENERAL_CACHE_PUBLIC), "expected EvEnd < EventSpaceEnd"); + + class TEvAskData: public NActors::TEventLocal { + private: + YDB_READONLY(EConsumer, Consumer, EConsumer::Undefined); + + bool AddressesExtracted = false; + THashSet Addresses; + + bool CallbackExtracted = false; + std::shared_ptr Callback; + + public: + TEvAskData(const EConsumer consumer, THashSet&& addresses, std::shared_ptr&& callback) + : Consumer(consumer) + , Addresses(std::move(addresses)) + , Callback(std::move(callback)) { + } + + THashSet ExtractAddresses() { + AFL_VERIFY(!AddressesExtracted); + AddressesExtracted = true; + return std::move(Addresses); + } + std::shared_ptr ExtractCallback() { + AFL_VERIFY(!CallbackExtracted); + CallbackExtracted = true; + return std::move(Callback); + } + }; +}; + +} // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/service.cpp b/ydb/core/tx/general_cache/usage/service.cpp new file mode 100644 index 000000000000..eb3953901e4b --- /dev/null +++ b/ydb/core/tx/general_cache/usage/service.cpp @@ -0,0 +1,5 @@ +#include "service.h" + +namespace NKikimr::NConveyorComposite { + +} diff --git a/ydb/core/tx/general_cache/usage/service.h b/ydb/core/tx/general_cache/usage/service.h new file mode 100644 index 000000000000..18c86c39dacc --- /dev/null +++ b/ydb/core/tx/general_cache/usage/service.h @@ -0,0 +1,45 @@ +#pragma once +#include "abstract.h" +#include "config.h" + +#include + +namespace NKikimr::NGeneralCache { + +template +class TServiceOperator { +private: + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + using ICallback = NPublic::ICallback; + + using TSelf = TServiceOperator; + +public: + static void AskObjects(const EConsumer consumer, THashSet&& addresses, std::shared_ptr&& callback) { + AFL_VERIFY(NActors::TlsActivationContext); + auto& context = NActors::TActorContext::AsActorContext(); + context.Send(GetCurrentNodeServiceId(), new NPublic::TEvents::TEvAskData(consumer, std::move(addresses), std::move(callback))); + } + static void ModifyObjects(THashMap&& add, THashSet&& remove) { + AFL_VERIFY(NActors::TlsActivationContext); + auto& context = NActors::TActorContext::AsActorContext(); + context.Send( + GetCurrentNodeServiceId(), new NSource::TEvents::TEvAdditionalObjectsInfo(std::move(add), std::move(remove))); + } + static NActors::TActorId MakeServiceId(const ui32 nodeId) { + return NActors::TActorId(nodeId, "SrvcCach" + TPolicy::GetServiceCode()); + } + static NActors::TActorId GetCurrentNodeServiceId() { + AFL_VERIFY(NActors::TlsActivationContext); + auto& context = NActors::TActorContext::AsActorContext(); + const NActors::TActorId& selfId = context.SelfID; + return MakeServiceId(selfId.NodeId()); + } + static NActors::IActor* CreateService(const NPublic::TConfig& config, TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) { + return new NPrivate::TDistributor(config, conveyorSignals); + } +}; + +} // namespace NKikimr::NGeneralCache diff --git a/ydb/core/tx/general_cache/usage/ya.make b/ydb/core/tx/general_cache/usage/ya.make new file mode 100644 index 000000000000..1a230bf7a0df --- /dev/null +++ b/ydb/core/tx/general_cache/usage/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + abstract.cpp + events.cpp + config.cpp + service.cpp +) + +PEERDIR( + ydb/core/protos + ydb/core/tx/general_cache/service + ydb/core/tx/general_cache/source +) + +END() diff --git a/ydb/core/tx/general_cache/ya.make b/ydb/core/tx/general_cache/ya.make new file mode 100644 index 000000000000..c8a17ad359b5 --- /dev/null +++ b/ydb/core/tx/general_cache/ya.make @@ -0,0 +1,12 @@ +LIBRARY() + +SRCS( +) + +PEERDIR( + ydb/core/tx/general_cache/service + ydb/core/tx/general_cache/source + ydb/core/tx/general_cache/usage +) + +END() diff --git a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp index e09db24f1f81..77e001447e76 100644 --- a/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp +++ b/ydb/core/tx/schemeshard/ut_olap/ut_olap.cpp @@ -852,9 +852,6 @@ Y_UNIT_TEST_SUITE(TOlap) { Y_UNIT_TEST(StoreStats) { TTestBasicRuntime runtime; - for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { - NKikimr::NTxUT::SetupSharedMetadataAccessorCacheService(runtime, node); - } TTestEnv env(runtime); runtime.SetLogPriority(NKikimrServices::TX_COLUMNSHARD, NActors::NLog::PRI_DEBUG); runtime.UpdateCurrentTime(TInstant::Now() - TDuration::Seconds(600)); @@ -1009,9 +1006,6 @@ Y_UNIT_TEST_SUITE(TOlap) { Y_UNIT_TEST(StoreStatsQuota) { TTestBasicRuntime runtime; - for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { - NKikimr::NTxUT::SetupSharedMetadataAccessorCacheService(runtime, node); - } TTestEnvOptions opts; opts.DisableStatsBatching(true); diff --git a/ydb/core/tx/ya.make b/ydb/core/tx/ya.make index 544315c8e567..dd0f6dc4f0e9 100644 --- a/ydb/core/tx/ya.make +++ b/ydb/core/tx/ya.make @@ -27,6 +27,7 @@ RECURSE( conveyor conveyor_composite columnshard + general_cache coordinator datashard locks diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index 452db7d0fa3e..7c582553eef1 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -386,6 +386,8 @@ enum EServiceKikimr { METADATA_MANAGER = 1502; METADATA_SECRET = 1503; + GENERAL_CACHE = 1504; + // Tiering TX_TIERING = 1600; From 0690efd2a31ded23618fb1e93e6efd6f15c234b1 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 26 Jun 2025 11:35:36 +0300 Subject: [PATCH 38/62] general cache requests abortion (#20190) Co-authored-by: ivanmorozov333 --- .../tx/columnshard/data_accessor/manager.h | 6 +- .../tx/columnshard/data_accessor/request.cpp | 1 - .../tx/columnshard/data_accessor/request.h | 192 ++---------------- .../engines/portions/data_accessor.h | 3 +- ydb/core/tx/general_cache/service/counters.h | 4 + ydb/core/tx/general_cache/service/manager.h | 15 +- ydb/core/tx/general_cache/usage/abstract.h | 4 + ydb/core/tx/general_cache/usage/service.h | 3 +- 8 files changed, 43 insertions(+), 185 deletions(-) diff --git a/ydb/core/tx/columnshard/data_accessor/manager.h b/ydb/core/tx/columnshard/data_accessor/manager.h index 573fe1e4d6af..21a816c61401 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/manager.h @@ -54,10 +54,14 @@ class TActorAccessorsManager: public IDataAccessorsManager { private: using TBase = IDataAccessorsManager; virtual void DoAskData(const std::shared_ptr& request) override { - class TAdapterCallback: public NKikimr::NGeneralCache::NPublic::ICallback { + class TAdapterCallback: public TGeneralCache::ICallback { private: std::shared_ptr AccessorsCallback; const ui64 RequestId; + virtual bool DoIsAborted() const override { + return AccessorsCallback->GetAbortionFlag() && AccessorsCallback->GetAbortionFlag()->Val(); + } + virtual void DoOnResultReady(THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errorAddresses) const override { diff --git a/ydb/core/tx/columnshard/data_accessor/request.cpp b/ydb/core/tx/columnshard/data_accessor/request.cpp index bdf3f6cf2af8..0b3717f6228f 100644 --- a/ydb/core/tx/columnshard/data_accessor/request.cpp +++ b/ydb/core/tx/columnshard/data_accessor/request.cpp @@ -3,7 +3,6 @@ namespace NKikimr::NOlap { void IDataAccessorRequestsSubscriber::RegisterRequestId(const TDataAccessorsRequest& request) { - AFL_VERIFY(!request.IsFetched()); AFL_VERIFY(RequestIds.emplace(request.GetRequestId()).second); } diff --git a/ydb/core/tx/columnshard/data_accessor/request.h b/ydb/core/tx/columnshard/data_accessor/request.h index 7d1852e96279..eaa3088e64e6 100644 --- a/ydb/core/tx/columnshard/data_accessor/request.h +++ b/ydb/core/tx/columnshard/data_accessor/request.h @@ -123,106 +123,15 @@ class TFakeDataAccessorsSubscriber: public IDataAccessorRequestsSubscriber { } }; -class TPathFetchingState { -public: - enum class EFetchStage { - Preparing, - Fetching, - Error, - Fetched - }; - -private: - const TInternalPathId PathId; - - YDB_READONLY(EFetchStage, Stage, EFetchStage::Preparing); - YDB_READONLY_DEF(TString, ErrorMessage); - THashMap Portions; - THashMap PortionAccessors; - -public: - TString DebugString() const { - TStringBuilder sb; - sb << "portions_count=" << Portions.size(); - return sb; - } - - explicit TPathFetchingState(const TInternalPathId pathId) - : PathId(pathId) { - } - - const THashMap& GetPortions() const { - return Portions; - } - - bool IsFinished() { - return Portions.empty() || Stage == EFetchStage::Error; - } - - THashMap&& DetachAccessors() { - return std::move(PortionAccessors); - } - - void AddPortion(const TPortionInfo::TConstPtr& portion) { - AFL_VERIFY(Stage == EFetchStage::Preparing); - AFL_VERIFY(portion->GetPathId() == PathId); - AFL_VERIFY(Portions.emplace(portion->GetPortionId(), portion).second); - } - - void AddAccessor( - const TPortionDataAccessor& accessor, const std::optional>& columnIds, const std::optional>& indexIds) { - AFL_VERIFY(Stage == EFetchStage::Fetching); - AFL_VERIFY(Portions.erase(accessor.GetPortionInfo().GetPortionId())); - AFL_VERIFY(PortionAccessors.emplace(accessor.GetPortionInfo().GetPortionId(), accessor.Extract(columnIds, indexIds)).second); - if (Portions.empty()) { - AFL_VERIFY(Stage == EFetchStage::Fetching); - Stage = EFetchStage::Fetched; - } - } - void StartFetch() { - AFL_VERIFY(Stage == EFetchStage::Preparing); - Stage = EFetchStage::Fetching; - AFL_VERIFY(Portions.size()); - } - - void OnError(const TString& errorMessage) { - AFL_VERIFY(Stage == EFetchStage::Fetching); - Stage = EFetchStage::Error; - ErrorMessage = errorMessage; - } -}; - class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter { private: static inline TAtomicCounter Counter = 0; - ui32 FetchStage = 0; YDB_READONLY(ui64, RequestId, Counter.Inc()); YDB_READONLY( NGeneralCache::TPortionsMetadataCachePolicy::EConsumer, Consumer, NGeneralCache::TPortionsMetadataCachePolicy::DefaultConsumer()); - THashSet PortionIds; - THashMap PathIdStatus; - THashSet PathIds; - TDataAccessorsResult AccessorsByPathId; - YDB_READONLY_DEF(std::optional>, ColumnIds); - std::optional> IndexIds; - - TAtomicCounter PreparingCount = 0; - TAtomicCounter FetchingCount = 0; - TAtomicCounter ReadyCount = 0; - + THashMap Portions; std::shared_ptr Subscriber; - void CheckReady() { - if (PathIdStatus.size()) { - return; - } - AFL_VERIFY(!PreparingCount.Val()); - AFL_VERIFY(!FetchingCount.Val()); - FetchStage = 2; - Subscriber->OnResult(RequestId, std::move(AccessorsByPathId)); - Subscriber = nullptr; - } - public: std::shared_ptr ExtractSubscriber() { AFL_VERIFY(HasSubscriber()); @@ -231,24 +140,17 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter BuildAddresses(const NActors::TActorId tabletActorId) const { THashSet result; - for (auto&& i : PathIdStatus) { - for (auto&& [_, p] : i.second.GetPortions()) { - AFL_VERIFY(result.emplace(NGeneralCache::TGlobalPortionAddress(tabletActorId, p->GetAddress())).second); - } + for (auto&& [_, p] : Portions) { + AFL_VERIFY(result.emplace(NGeneralCache::TGlobalPortionAddress(tabletActorId, p->GetAddress())).second); } return result; } - void SetColumnIds(const std::set& columnIds) { - AFL_VERIFY(!ColumnIds); - ColumnIds = columnIds; - } - TString DebugString() const { TStringBuilder sb; sb << "request_id=" << RequestId << ";"; - for (auto&& i : PathIdStatus) { - sb << i.first << "={" << i.second.DebugString() << "};"; + for (auto&& [id, p] : Portions) { + sb << id << "={" << p->DebugString() << "};"; } return sb; } @@ -259,10 +161,8 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounterPredictAccessorsMemory(schema); - } + for (auto&& [_, p] : Portions) { + result += p->PredictAccessorsMemory(schema); } return result; } @@ -283,93 +183,25 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter& GetPathIds() const { - return PathIds; + bool IsEmpty() const { + return Portions.empty(); } - bool IsEmpty() const { - return PortionIds.empty(); + void SetColumnIds(const std::set& /*columnIds*/) { } void RegisterSubscriber(const std::shared_ptr& subscriber) { AFL_VERIFY(!Subscriber); - AFL_VERIFY(FetchStage == 0); Subscriber = subscriber; Subscriber->RegisterRequestId(*this); } - const THashMap& StartFetching(const TInternalPathId pathId) { - AFL_VERIFY(!!Subscriber); - AFL_VERIFY(FetchStage <= 1); - FetchStage = 1; - - auto it = PathIdStatus.find(pathId); - AFL_VERIFY(it != PathIdStatus.end()); - it->second.StartFetch(); - - FetchingCount.Inc(); - AFL_VERIFY(PreparingCount.Dec() >= 0); - - return it->second.GetPortions(); - } - void AddPortion(const TPortionInfo::TConstPtr& portion) { AFL_VERIFY(portion); - AFL_VERIFY(FetchStage <= 1); - AFL_VERIFY(PortionIds.emplace(portion->GetPortionId()).second); - const auto& pathId = portion->GetPathId(); - PathIds.emplace(pathId); - auto it = PathIdStatus.find(pathId); - if (it == PathIdStatus.end()) { - PreparingCount.Inc(); - it = PathIdStatus.emplace(pathId, TPathFetchingState{ pathId }).first; - } - it->second.AddPortion(portion); - } - - bool IsFetched() const { - return FetchStage == 2; - } - - void AddError(const TInternalPathId pathId, const TString& errorMessage) { - AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", errorMessage)("event", "ErrorOnFetching")("path_id", pathId); - AFL_VERIFY(FetchStage <= 1); - auto itStatus = PathIdStatus.find(pathId); - AFL_VERIFY(itStatus != PathIdStatus.end()); - itStatus->second.OnError(errorMessage); - PathIdStatus.erase(itStatus); - AFL_VERIFY(FetchingCount.Dec() >= 0); - ReadyCount.Inc(); - AccessorsByPathId.AddError(pathId, errorMessage); - CheckReady(); - } - - void AddAccessor(const TPortionDataAccessor& accessor) { - AFL_VERIFY(FetchStage == 1); - auto pathId = accessor.GetPortionInfo().GetPathId(); - { - auto itStatus = PathIdStatus.find(pathId); - AFL_VERIFY(itStatus != PathIdStatus.end()); - itStatus->second.AddAccessor(accessor, ColumnIds, IndexIds); - if (itStatus->second.IsFinished()) { - AFL_VERIFY(FetchingCount.Dec() >= 0); - ReadyCount.Inc(); - AccessorsByPathId.AddData(itStatus->second.DetachAccessors()); - PathIdStatus.erase(itStatus); - } - } - CheckReady(); - } - - void AddData(THashMap>&& accessors) { - for (auto&& i : accessors) { - for (auto&& a : i.second) { - AddAccessor(std::move(a)); - } - } + AFL_VERIFY(Portions.emplace(portion->GetPortionId(), portion).second); } TString GetTaskId() const { diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index cdb9723dd721..0d96af82e5ac 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -239,7 +239,8 @@ class TPortionDataAccessor { ui64 GetMetadataSize() const { return (Records ? (Records->size() * sizeof(TColumnRecord)) : 0) + - (Indexes ? (Indexes->size() * sizeof(TIndexChunk)) : 0); + (Indexes ? (Indexes->size() * sizeof(TIndexChunk)) : 0) + + sizeof(TPortionInfo::TConstPtr); } class TExtractContext { diff --git a/ydb/core/tx/general_cache/service/counters.h b/ydb/core/tx/general_cache/service/counters.h index d42d1fa35b94..12c012e62c80 100644 --- a/ydb/core/tx/general_cache/service/counters.h +++ b/ydb/core/tx/general_cache/service/counters.h @@ -24,7 +24,9 @@ class TManagerCounters: public NColumnShard::TCommonCountersOwner { const NMonitoring::TDynamicCounters::TCounterPtr RequestsQueueSize; const NMonitoring::TDynamicCounters::TCounterPtr ObjectsQueueSize; const NMonitoring::TDynamicCounters::TCounterPtr IncomingRequestsCount; + const NMonitoring::TDynamicCounters::TCounterPtr IncomingAbortedRequestsCount; const NMonitoring::TDynamicCounters::TCounterPtr DirectRequests; + const NMonitoring::TDynamicCounters::TCounterPtr AbortedRequests; const NMonitoring::TDynamicCounters::TCounterPtr AdditionalObjectInfo; const NMonitoring::TDynamicCounters::TCounterPtr RemovedObjectInfo; const NMonitoring::TDynamicCounters::TCounterPtr FetchedObject; @@ -45,7 +47,9 @@ class TManagerCounters: public NColumnShard::TCommonCountersOwner { , RequestsQueueSize(TBase::GetValue("RequestsQueue/Size/Count")) , ObjectsQueueSize(TBase::GetValue("ObjectsQueue/Size/Count")) , IncomingRequestsCount(TBase::GetDeriviative("Incoming/Requests/Count")) + , IncomingAbortedRequestsCount(TBase::GetDeriviative("Incoming/AbortedRequests/Count")) , DirectRequests(TBase::GetDeriviative("DirectRequest/Count")) + , AbortedRequests(TBase::GetDeriviative("AbortedRequest/Count")) , AdditionalObjectInfo(TBase::GetDeriviative("AdditionalInfo/Count")) , RemovedObjectInfo(TBase::GetDeriviative("RemovedInfo/Count")) , FetchedObject(TBase::GetDeriviative("DirectRequest/Fetched/Count")) diff --git a/ydb/core/tx/general_cache/service/manager.h b/ydb/core/tx/general_cache/service/manager.h index fade980bd1c5..07f405e83c4c 100644 --- a/ydb/core/tx/general_cache/service/manager.h +++ b/ydb/core/tx/general_cache/service/manager.h @@ -31,6 +31,10 @@ class TRequest: public NColumnShard::TMonitoringObjectsCounter const EConsumer Consumer; public: + bool IsAborted() const { + return Callback->IsAborted(); + } + EConsumer GetConsumer() const { return Consumer; } @@ -93,6 +97,10 @@ class TManager { auto request = std::move(RequestsQueue.front()); RequestsQueue.pop_front(); QueueObjectsCount.Sub(request->GetWait().size()); + if (request->IsAborted()) { + Counters->AbortedRequests->Inc(); + continue; + } AFL_VERIFY(RequestsInProgress.emplace(request->GetRequestId()).second); auto& addresses = requestedAddresses[request->GetConsumer()]; for (auto&& i : request->GetWait()) { @@ -169,7 +177,12 @@ class TManager { void AddRequest(const std::shared_ptr& request) { AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "add_request"); THashMap objectsResult; - Counters->IncomingRequestsCount->Inc(); + if (request->IsAborted()) { + Counters->IncomingAbortedRequestsCount->Inc(); + return; + } else { + Counters->IncomingRequestsCount->Inc(); + } for (auto&& i : request->GetWait()) { auto it = Cache.Find(i); if (it == Cache.End()) { diff --git a/ydb/core/tx/general_cache/usage/abstract.h b/ydb/core/tx/general_cache/usage/abstract.h index 2766d0ffb72f..ca35131f1d00 100644 --- a/ydb/core/tx/general_cache/usage/abstract.h +++ b/ydb/core/tx/general_cache/usage/abstract.h @@ -12,10 +12,14 @@ class ICallback { virtual void DoOnResultReady(THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errorAddresses) const = 0; + virtual bool DoIsAborted() const = 0; public: virtual ~ICallback() = default; + bool IsAborted() const { + return DoIsAborted(); + } void OnResultReady(THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errorAddresses) const { DoOnResultReady(std::move(objectAddresses), std::move(removedAddresses), std::move(errorAddresses)); diff --git a/ydb/core/tx/general_cache/usage/service.h b/ydb/core/tx/general_cache/usage/service.h index 18c86c39dacc..a9ace52f1af4 100644 --- a/ydb/core/tx/general_cache/usage/service.h +++ b/ydb/core/tx/general_cache/usage/service.h @@ -8,12 +8,13 @@ namespace NKikimr::NGeneralCache { template class TServiceOperator { -private: +public: using TAddress = typename TPolicy::TAddress; using TObject = typename TPolicy::TObject; using EConsumer = typename TPolicy::EConsumer; using ICallback = NPublic::ICallback; +private: using TSelf = TServiceOperator; public: From 9c266411511ee18ebe346a8315e598d5c09177db Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 26 Jun 2025 11:39:01 +0300 Subject: [PATCH 39/62] correct metadata control (#20070) Co-authored-by: ivanmorozov333 --- ydb/core/protos/data_events.proto | 1 + ydb/core/protos/flat_scheme_op.proto | 1 + .../tx/columnshard/columnshard__write.cpp | 13 +++-- ydb/core/tx/columnshard/common/path_id.h | 8 +++ .../tx/columnshard/counters/columnshard.h | 5 +- .../data_accessor/abstract/collector.h | 5 ++ .../storage/optimizer/abstract/counters.cpp | 5 ++ .../storage/optimizer/abstract/counters.h | 44 +++++++++++++++ .../storage/optimizer/abstract/optimizer.h | 43 +++++++++++++- .../storage/optimizer/abstract/ya.make | 1 + .../lbuckets/constructor/constructor.cpp | 2 +- .../optimizer/lbuckets/planner/optimizer.h | 4 +- .../lcbuckets/constructor/constructor.cpp | 5 +- .../constructor/level/zero_level.cpp | 1 + .../lcbuckets/planner/level/abstract.h | 9 ++- .../lcbuckets/planner/level/counters.h | 8 ++- .../optimizer/lcbuckets/planner/optimizer.cpp | 6 +- .../optimizer/lcbuckets/planner/optimizer.h | 4 +- .../storage/optimizer/tiling/tiling.cpp | 11 +++- .../tx/columnshard/tablet/write_queue.cpp | 56 +++++++++++-------- ydb/core/tx/columnshard/tablet/write_queue.h | 29 +++++++--- ydb/core/tx/data_events/shard_writer.cpp | 3 + 22 files changed, 210 insertions(+), 54 deletions(-) create mode 100644 ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.cpp create mode 100644 ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.h diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto index 0010bd0c6d0f..9a28e8843137 100644 --- a/ydb/core/protos/data_events.proto +++ b/ydb/core/protos/data_events.proto @@ -123,6 +123,7 @@ message TEvWrite { optional uint32 GranuleShardingVersionId = 9; optional ELockMode LockMode = 10; + optional uint64 TimeoutSeconds = 11; } message TEvWriteResult { diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto index 041c14d2edbb..22ee6ef5af9c 100644 --- a/ydb/core/protos/flat_scheme_op.proto +++ b/ydb/core/protos/flat_scheme_op.proto @@ -571,6 +571,7 @@ message TCompactionSelectorConstructorContainer { message TCompactionPlannerConstructorContainer { optional string ClassName = 1; + optional uint64 NodePortionsCountLimit = 2; message TLOptimizer { diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index c67aabd81139..fce4c14e3a24 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -275,8 +275,8 @@ class TProposeWriteTransaction: public TExtendedTransactionBase { } proto.SetLockId(WriteCommit->GetLockId()); TxOperator = Self->GetProgressTxController().StartProposeOnExecute( - TTxController::TTxInfo(kind, WriteCommit->GetTxId(), Source, Self->GetProgressTxController().GetAllowedStep(), - Cookie, {}), proto.SerializeAsString(), txc); + TTxController::TTxInfo(kind, WriteCommit->GetTxId(), Source, Self->GetProgressTxController().GetAllowedStep(), Cookie, {}), + proto.SerializeAsString(), txc); return true; } @@ -334,6 +334,10 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor const auto& record = ev->Get()->Record; const auto source = ev->Sender; const auto cookie = ev->Cookie; + std::optional writeTimeout; + if (record.HasTimeoutSeconds()) { + writeTimeout = TDuration::Seconds(record.GetTimeoutSeconds()); + } if (!TablesManager.GetPrimaryIndex()) { Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL); @@ -427,7 +431,7 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor const auto schemeShardLocalPathId = TSchemeShardLocalPathId::FromProto(operation.GetTableId()); const auto& internalPathId = TablesManager.ResolveInternalPathId(schemeShardLocalPathId); AFL_VERIFY(internalPathId); - const auto& pathId = TUnifiedPathId{*internalPathId, schemeShardLocalPathId}; + const auto& pathId = TUnifiedPathId{ *internalPathId, schemeShardLocalPathId }; if (!TablesManager.IsReadyForStartWrite(*internalPathId, false)) { sendError("table not writable", NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR); return; @@ -474,7 +478,8 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor } Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize()); - WriteTasksQueue->Enqueue(TWriteTask(arrowData, schema, source, granuleShardingVersionId, pathId, cookie, lockId, *mType, behaviour)); + WriteTasksQueue->Enqueue(TWriteTask( + arrowData, schema, source, granuleShardingVersionId, pathId, cookie, lockId, *mType, behaviour, writeTimeout, record.GetTxId())); WriteTasksQueue->Drain(false, ctx); } diff --git a/ydb/core/tx/columnshard/common/path_id.h b/ydb/core/tx/columnshard/common/path_id.h index 207ffae747b2..7e0b27193f60 100644 --- a/ydb/core/tx/columnshard/common/path_id.h +++ b/ydb/core/tx/columnshard/common/path_id.h @@ -86,6 +86,14 @@ static_assert(sizeof(TSchemeShardLocalPathId) == sizeof(ui64)); struct TUnifiedPathId { TInternalPathId InternalPathId; TSchemeShardLocalPathId SchemeShardLocalPathId; + + const TInternalPathId& GetInternalPathId() const { + return InternalPathId; + } + const TSchemeShardLocalPathId& GetSchemeShardLocalPathId() const { + return SchemeShardLocalPathId; + } + explicit operator bool() const { return InternalPathId && SchemeShardLocalPathId; } diff --git a/ydb/core/tx/columnshard/counters/columnshard.h b/ydb/core/tx/columnshard/counters/columnshard.h index 0716e21d4314..bf1af7e763d1 100644 --- a/ydb/core/tx/columnshard/counters/columnshard.h +++ b/ydb/core/tx/columnshard/counters/columnshard.h @@ -44,6 +44,7 @@ class TWriteCounters: public TCommonCountersOwner { public: const NMonitoring::TDynamicCounters::TCounterPtr QueueWaitSize; + const NMonitoring::TDynamicCounters::TCounterPtr TimeoutRate; void OnWritingTaskDequeue(const TDuration d) { HistogramDurationQueueWait->Collect(d.MilliSeconds()); @@ -52,7 +53,9 @@ class TWriteCounters: public TCommonCountersOwner { TWriteCounters(TCommonCountersOwner& owner) : TBase(owner, "activity", "writing") , WriteFlowCounters(std::make_shared()) - , QueueWaitSize(TBase::GetValue("Write/Queue/Size")) { + , QueueWaitSize(TBase::GetValue("Write/Queue/Size")) + , TimeoutRate(TBase::GetDeriviative("Write/Timeout/Count")) + { VolumeWriteData = TBase::GetDeriviative("Write/Incoming/Bytes"); HistogramBytesWriteDataCount = TBase::GetHistogram("Write/Incoming/ByBytes/Count", NMonitoring::ExponentialHistogram(18, 2, 100)); HistogramBytesWriteDataBytes = TBase::GetHistogram("Write/Incoming/ByBytes/Bytes", NMonitoring::ExponentialHistogram(18, 2, 100)); diff --git a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h index 48d6fd59f205..e61d15811eda 100644 --- a/ydb/core/tx/columnshard/data_accessor/abstract/collector.h +++ b/ydb/core/tx/columnshard/data_accessor/abstract/collector.h @@ -9,6 +9,11 @@ class TGranuleMeta; class TPortionInfo; } +namespace NKikimr::NOlap { +class TGranuleMeta; +class TPortionInfo; +} + namespace NKikimr::NOlap::NDataAccessorControl { class IAccessorCallback { public: diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.cpp new file mode 100644 index 000000000000..f7252baaa0c2 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.cpp @@ -0,0 +1,5 @@ +#include "counters.h" + +namespace NKikimr::NOlap::NStorageOptimizer { + +} diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.h new file mode 100644 index 000000000000..dd4a5bdc8642 --- /dev/null +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/counters.h @@ -0,0 +1,44 @@ +#pragma once +#include +#include + +#include + +namespace NKikimr::NOlap::NStorageOptimizer { + +class TGlobalCounters: public NColumnShard::TCommonCountersOwner { +private: + using TBase = NColumnShard::TCommonCountersOwner; + const NMonitoring::TDynamicCounters::TCounterPtr NodePortionsCount; + const NMonitoring::TDynamicCounters::TCounterPtr NodePortionsCountLimit; + +public: + TGlobalCounters() + : TBase("CompactionOptimizer") + , NodePortionsCount(TBase::GetValue("Node/Portions/Count")) + , NodePortionsCountLimit(TBase::GetValue("Node/Portions/Limit/Count")) + { + } + + static const NMonitoring::TDynamicCounters::TCounterPtr& GetNodePortionsCount() { + return Singleton()->NodePortionsCount; + } + + static const NMonitoring::TDynamicCounters::TCounterPtr& GetNodePortionsCountLimit() { + return Singleton()->NodePortionsCountLimit; + } +}; + +class TCounters { +public: + const NMonitoring::TDynamicCounters::TCounterPtr NodePortionsCount; + const NMonitoring::TDynamicCounters::TCounterPtr NodePortionsCountLimit; + + TCounters() + : NodePortionsCount(TGlobalCounters::GetNodePortionsCount()) + , NodePortionsCountLimit(TGlobalCounters::GetNodePortionsCountLimit()) + { + } +}; + +} // namespace NKikimr::NOlap::NStorageOptimizer diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h index 458a91012080..9c8f3f9aeaea 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/optimizer.h @@ -1,9 +1,12 @@ #pragma once +#include "counters.h" + #include #include #include #include +#include #include #include @@ -91,6 +94,10 @@ class IOptimizerPlanner { virtual bool DoIsOverloaded() const { return false; } + const ui32 NodePortionsCountLimit = 0; + static inline TAtomicCounter NodePortionsCounter = 0; + TPositiveControlInteger LocalPortionsCount; + std::shared_ptr Counters = std::make_shared(); protected: virtual void DoModifyPortions( @@ -116,10 +123,15 @@ class IOptimizerPlanner { return baseLevel; } - IOptimizerPlanner(const TInternalPathId pathId) - : PathId(pathId) { + IOptimizerPlanner(const TInternalPathId pathId, const ui32 nodePortionsCountLimit) + : PathId(pathId) + , NodePortionsCountLimit(nodePortionsCountLimit) { + Counters->NodePortionsCountLimit->Set(NodePortionsCountLimit); } bool IsOverloaded() const { + if (NodePortionsCountLimit <= NodePortionsCounter.Val()) { + return true; + } return DoIsOverloaded(); } TConclusionStatus CheckWriteData() const { @@ -152,7 +164,10 @@ class IOptimizerPlanner { return TModificationGuard(*this); } - virtual ~IOptimizerPlanner() = default; + virtual ~IOptimizerPlanner() { + NodePortionsCounter.Sub(LocalPortionsCount.Val()); + Counters->NodePortionsCount->Set(NodePortionsCounter.Val()); + } TString DebugString() const { return DoDebugString(); } @@ -165,6 +180,11 @@ class IOptimizerPlanner { void ModifyPortions(const THashMap>& add, const THashMap>& remove) { NActors::TLogContextGuard g(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("path_id", PathId)); + LocalPortionsCount.Add(add.size()); + LocalPortionsCount.Sub(remove.size()); + NodePortionsCounter.Add(add.size()); + NodePortionsCounter.Sub(remove.size()); + Counters->NodePortionsCount->Set(NodePortionsCounter.Val()); DoModifyPortions(add, remove); } @@ -207,6 +227,8 @@ class IOptimizerPlannerConstructor { using TProto = NKikimrSchemeOp::TCompactionPlannerConstructorContainer; private: + ui32 NodePortionsCountLimit = 1000000; + virtual TConclusion> DoBuildPlanner(const TBuildContext& context) const = 0; virtual void DoSerializeToProto(TProto& proto) const = 0; virtual bool DoDeserializeFromProto(const TProto& proto) = 0; @@ -214,6 +236,10 @@ class IOptimizerPlannerConstructor { virtual bool DoApplyToCurrentObject(IOptimizerPlanner& current) const = 0; public: + ui32 GetNodePortionsCountLimit() const { + return NodePortionsCountLimit; + } + static std::shared_ptr BuildDefault() { auto result = TFactory::MakeHolder("lc-buckets"); AFL_VERIFY(!!result); @@ -230,6 +256,13 @@ class IOptimizerPlannerConstructor { } TConclusionStatus DeserializeFromJson(const NJson::TJsonValue& jsonInfo) { + if (jsonInfo.Has("node_portions_count_limit")) { + const auto& jsonValue = jsonInfo["node_portions_count_limit"]; + if (!jsonValue.IsUInteger()) { + return TConclusionStatus::Fail("incorrect node_portions_count_limit value have to be unsigned int"); + } + NodePortionsCountLimit = jsonValue.GetUInteger(); + } return DoDeserializeFromJson(jsonInfo); } @@ -239,6 +272,7 @@ class IOptimizerPlannerConstructor { virtual TString GetClassName() const = 0; void SerializeToProto(TProto& proto) const { + proto.SetNodePortionsCountLimit(NodePortionsCountLimit); DoSerializeToProto(proto); } @@ -255,6 +289,9 @@ class IOptimizerPlannerConstructor { } bool DeserializeFromProto(const TProto& proto) { + if (proto.HasNodePortionsCountLimit()) { + NodePortionsCountLimit = proto.GetNodePortionsCountLimit(); + } return DoDeserializeFromProto(proto); } }; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/ya.make b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/ya.make index 140b1ed351bf..3f96a5717477 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/ya.make +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/abstract/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( optimizer.cpp + counters.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/constructor/constructor.cpp index c66583b7647e..e509f9a8e3d2 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/constructor/constructor.cpp @@ -4,7 +4,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLBuckets { NKikimr::TConclusion> TOptimizerPlannerConstructor::DoBuildPlanner(const TBuildContext& context) const { - return std::make_shared(context.GetPathId(), context.GetStorages(), context.GetPKSchema()); + return std::make_shared(context.GetPathId(), context.GetStorages(), context.GetPKSchema(), GetNodePortionsCountLimit()); } bool TOptimizerPlannerConstructor::DoApplyToCurrentObject(IOptimizerPlanner& current) const { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h index fd886db94103..8ca913e0d273 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/planner/optimizer.h @@ -1280,8 +1280,8 @@ class TOptimizerPlanner: public IOptimizerPlanner { } TOptimizerPlanner(const TInternalPathId pathId, const std::shared_ptr& storagesManager, - const std::shared_ptr& primaryKeysSchema) - : TBase(pathId) + const std::shared_ptr& primaryKeysSchema, const ui64 nodePortionsCountLimit) + : TBase(pathId, nodePortionsCountLimit) , Counters(std::make_shared()) , Buckets(primaryKeysSchema, storagesManager, Counters) , StoragesManager(storagesManager) { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp index e5c14d301707..8ae59b8bab65 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/constructor.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { -NKikimr::TConclusion> TOptimizerPlannerConstructor::DoBuildPlanner(const TBuildContext& context) const { +TConclusion> TOptimizerPlannerConstructor::DoBuildPlanner(const TBuildContext& context) const { auto counters = std::make_shared(); auto portionsInfo = std::make_shared(); const TString defaultSelectorName = "default"; @@ -114,7 +114,8 @@ NKikimr::TConclusion(context.GetPathId(), context.GetStorages(), context.GetPKSchema(), counters, portionsInfo, std::move(levels), std::move(selectors)); + return std::make_shared(context.GetPathId(), context.GetStorages(), context.GetPKSchema(), counters, portionsInfo, + std::move(levels), std::move(selectors), GetNodePortionsCountLimit()); } bool TOptimizerPlannerConstructor::DoApplyToCurrentObject(IOptimizerPlanner& /*current*/) const { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp index 2ce33e03a724..17b79af38763 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/constructor/level/zero_level.cpp @@ -40,6 +40,7 @@ TConclusionStatus TZeroLevelConstructor::DoDeserializeFromJson(const NJson::TJso } PortionsCountLimit = jsonValue.GetUInteger(); } + if (json.Has("portions_size_limit")) { const auto& jsonValue = json["portions_size_limit"]; if (!jsonValue.IsUInteger()) { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/abstract.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/abstract.h index 492fa5fe91c7..adcdb18e4ef7 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/abstract.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/abstract.h @@ -321,7 +321,8 @@ class TLimitsOverloadChecker: public IOverloadChecker { class IPortionsLevel { private: - virtual std::vector DoModifyPortions(const std::vector& add, const std::vector& remove) = 0; + virtual std::vector DoModifyPortions( + const std::vector& add, const std::vector& remove) = 0; virtual ui64 DoGetWeight() const = 0; virtual TInstant DoGetWeightExpirationInstant() const = 0; virtual NArrow::NMerger::TIntervalPositions DoGetBucketPositions(const std::shared_ptr& pkSchema) const = 0; @@ -393,7 +394,8 @@ class IPortionsLevel { return NextLevel; } - virtual ~IPortionsLevel() = default; + virtual ~IPortionsLevel() { + } IPortionsLevel(const ui64 levelId, const std::shared_ptr& nextLevel, const std::shared_ptr& overloadChecker, const TLevelCounters levelCounters, const std::vector>& selectors, const TString& defaultSelectorName) @@ -451,7 +453,8 @@ class IPortionsLevel { return DoGetAffectedPortionBytes(from, to); } - [[nodiscard]] std::vector ModifyPortions(const std::vector& add, const std::vector& remove) { + [[nodiscard]] std::vector ModifyPortions( + const std::vector& add, const std::vector& remove) { std::vector addSelective; std::vector removeSelective; for (ui32 idx = 0; idx < Selectors.size(); ++idx) { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/counters.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/counters.h index 95d4924b3ece..9a83c91a7c86 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/counters.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/level/counters.h @@ -35,9 +35,13 @@ class TGlobalCounters: public NColumnShard::TCommonCountersOwner { } } - static std::shared_ptr BuildPortionsCounter(const ui32 levelId) { + static std::shared_ptr GetLevelAgents(const ui32 levelId) { AFL_VERIFY(levelId < Singleton()->Levels.size()); - return std::make_shared(*Singleton()->Levels[levelId]->Portions); + return Singleton()->Levels[levelId]; + } + + static std::shared_ptr BuildPortionsCounter(const ui32 levelId) { + return std::make_shared(*GetLevelAgents(levelId)->Portions); } }; diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp index a42d6eee7b86..576e8a4ef1d1 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.cpp @@ -12,9 +12,9 @@ namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets { TOptimizerPlanner::TOptimizerPlanner(const TInternalPathId pathId, const std::shared_ptr& storagesManager, - const std::shared_ptr& primaryKeysSchema, std::shared_ptr counters, std::shared_ptr portionsGroupInfo, - std::vector>&& levels, std::vector>&& selectors) - : TBase(pathId) + const std::shared_ptr& primaryKeysSchema, std::shared_ptr counters, std::shared_ptr portionsGroupInfo, std::vector>&& levels, + std::vector>&& selectors, const ui32 nodePortionsCountLimit) + : TBase(pathId, nodePortionsCountLimit) , Counters(counters) , PortionsInfo(portionsGroupInfo) , Selectors(std::move(selectors)) diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h index 880fd04c9ce9..4dd80a0e1489 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h @@ -151,9 +151,11 @@ class TOptimizerPlanner: public IOptimizerPlanner { return result; } + ~TOptimizerPlanner() = default; + TOptimizerPlanner(const TInternalPathId pathId, const std::shared_ptr& storagesManager, const std::shared_ptr& primaryKeysSchema, std::shared_ptr counters, std::shared_ptr portionsGroupInfo, - std::vector>&& levels, std::vector>&& selectors); + std::vector>&& levels, std::vector>&& selectors, const ui32 nodePortionsCountLimit); }; } // namespace NKikimr::NOlap::NStorageOptimizer::NLCBuckets diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp index d8f1e38e3628..6d4cc41efa1d 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp @@ -30,6 +30,7 @@ struct TSettings { ui32 FullCompactionUntilLevel = 0; ui64 FullCompactionMaxBytes = -1; bool CompactNextLevelEdges = false; + ui64 NodePortionsCountLimit = 1000000; NJson::TJsonValue SettingsJson; @@ -444,7 +445,7 @@ class TOptimizerPlanner : public IOptimizerPlanner, private TSettings { public: TOptimizerPlanner(const TInternalPathId pathId, const std::shared_ptr& storagesManager, const std::shared_ptr& primaryKeysSchema, const TSettings& settings = {}) - : TBase(pathId) + : TBase(pathId, settings.NodePortionsCountLimit) , TSettings(settings) , StoragesManager(storagesManager) , PrimaryKeysSchema(primaryKeysSchema) @@ -710,11 +711,17 @@ class TOptimizerPlannerConstructor : public IOptimizerPlannerConstructor { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("error", "cannot parse tiling compaction optimizer from proto")("description", status.GetErrorDescription()); return false; } + Settings.NodePortionsCountLimit = GetNodePortionsCountLimit(); return true; } TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonInfo) override { - return Settings.DeserializeFromJson(jsonInfo); + auto conclusion = Settings.DeserializeFromJson(jsonInfo); + if (conclusion.IsFail()) { + return conclusion; + } + Settings.NodePortionsCountLimit = GetNodePortionsCountLimit(); + return TConclusionStatus::Success(); } bool DoApplyToCurrentObject(IOptimizerPlanner& current) const override { diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 9e04423d46d5..1f7d3a6c3f9b 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NColumnShard { -bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { +bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) const { owner->Counters.GetCSCounters().WritingCounters->OnWritingTaskDequeue(TMonotonic::Now() - Created); owner->OperationsManager->RegisterLock(LockId, owner->Generation()); auto writeOperation = owner->OperationsManager->CreateWriteOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType); @@ -26,29 +26,41 @@ bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) { return true; } +void TWriteTask::Abort(TColumnShard* owner, const TString& reason, const TActorContext& ctx) const { + auto result = NEvents::TDataEvents::TEvWriteResult::BuildError( + owner->TabletID(), TxId, NKikimrDataEvents::TEvWriteResult::STATUS_INTERNAL_ERROR, reason); + owner->Counters.GetWritesMonitor()->OnFinishWrite(ArrowData->GetSize()); + ctx.Send(SourceId, result.release(), 0, Cookie); +} + bool TWriteTasksQueue::Drain(const bool onWakeup, const TActorContext& ctx) { if (onWakeup) { WriteTasksOverloadCheckerScheduled = false; } - std::vector toRemove; ui32 countTasks = 0; - for (auto&& i : WriteTasks) { - auto overloadStatus = Owner->CheckOverloadedWait(i.first); - if (overloadStatus != TColumnShard::EOverloadStatus::None) { - Owner->Counters.GetCSCounters().OnWaitingOverload(overloadStatus); - countTasks += i.second.size(); - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "wait_overload")("status", overloadStatus)("path_id", i.first)( - "size", i.second.size()); - continue; - } - for (auto&& t : i.second) { - t.Execute(Owner, ctx); + const TMonotonic now = TMonotonic::Now(); + std::set overloaded; + for (auto it = WriteTasks.begin(); it != WriteTasks.end();) { + if (it->IsDeprecated(now)) { + it->Abort(Owner, "timeout", ctx); + Owner->Counters.GetCSCounters().WritingCounters->TimeoutRate->Inc(); + it = WriteTasks.erase(it); + } else if (!overloaded.contains(it->GetInternalPathId())) { + auto overloadStatus = Owner->CheckOverloadedWait(it->GetInternalPathId()); + if (overloadStatus != TColumnShard::EOverloadStatus::None) { + overloaded.emplace(it->GetInternalPathId()); + Owner->Counters.GetCSCounters().OnWaitingOverload(overloadStatus); + ++countTasks; + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "wait_overload")("status", overloadStatus)( + "path_id", it->GetInternalPathId()); + ++it; + } else { + it->Execute(Owner, ctx); + it = WriteTasks.erase(it); + } + } else { + ++it; } - toRemove.emplace_back(i.first); - } - - for (auto&& i : toRemove) { - AFL_VERIFY(WriteTasks.erase(i)); } if (countTasks && !WriteTasksOverloadCheckerScheduled) { @@ -56,18 +68,16 @@ bool TWriteTasksQueue::Drain(const bool onWakeup, const TActorContext& ctx) { WriteTasksOverloadCheckerScheduled = true; AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "queue_on_write")("size", countTasks); } - Owner->Counters.GetCSCounters().WritingCounters->QueueWaitSize->Add((i64)countTasks - PredWriteTasksSize); - PredWriteTasksSize = (i64)countTasks; + Owner->Counters.GetCSCounters().WritingCounters->QueueWaitSize->Set(WriteTasks.size()); return !countTasks; } void TWriteTasksQueue::Enqueue(TWriteTask&& task) { - const auto& pathId = task.GetInternalPathId(); - WriteTasks[pathId].emplace_back(std::move(task)); + WriteTasks.emplace(std::move(task)); } TWriteTasksQueue::~TWriteTasksQueue() { - Owner->Counters.GetCSCounters().WritingCounters->QueueWaitSize->Sub(PredWriteTasksSize); + Owner->Counters.GetCSCounters().WritingCounters->QueueWaitSize->Sub(WriteTasks.size()); } } // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/tablet/write_queue.h b/ydb/core/tx/columnshard/tablet/write_queue.h index e86aacf33f88..0646d8edf33e 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.h +++ b/ydb/core/tx/columnshard/tablet/write_queue.h @@ -1,16 +1,18 @@ #pragma once +#include #include #include -#include #include namespace NKikimr::NColumnShard { class TColumnShard; class TArrowData; -class TWriteTask: TMoveOnly { +class TWriteTask: public TMoveOnly { private: std::shared_ptr ArrowData; NOlap::ISnapshotSchema::TPtr Schema; + static inline TAtomicCounter Counter = 0; + const ui64 TaskId = Counter.Inc(); const NActors::TActorId SourceId; const std::optional GranuleShardingVersionId; const TUnifiedPathId PathId; @@ -19,11 +21,21 @@ class TWriteTask: TMoveOnly { const NEvWrite::EModificationType ModificationType; const EOperationBehaviour Behaviour; const TMonotonic Created = TMonotonic::Now(); + const std::optional Timeout; + const ui64 TxId; public: + bool operator<(const TWriteTask& item) const { + return std::tie(Created, PathId.GetInternalPathId(), TaskId) < std::tie(item.Created, item.PathId.GetInternalPathId(), item.TaskId); + } + + bool IsDeprecated(const TMonotonic now) const { + return Timeout ? (Created + *Timeout <= now) : false; + } + TWriteTask(const std::shared_ptr& arrowData, const NOlap::ISnapshotSchema::TPtr& schema, const NActors::TActorId sourceId, const std::optional& granuleShardingVersionId, const TUnifiedPathId pathId, const ui64 cookie, const ui64 lockId, - const NEvWrite::EModificationType modificationType, const EOperationBehaviour behaviour) + const NEvWrite::EModificationType modificationType, const EOperationBehaviour behaviour, const std::optional timeout, const ui64 txId) : ArrowData(arrowData) , Schema(schema) , SourceId(sourceId) @@ -32,7 +44,10 @@ class TWriteTask: TMoveOnly { , Cookie(cookie) , LockId(lockId) , ModificationType(modificationType) - , Behaviour(behaviour) { + , Behaviour(behaviour) + , Timeout(timeout) + , TxId(txId) + { } const TInternalPathId& GetInternalPathId() const { @@ -43,14 +58,14 @@ class TWriteTask: TMoveOnly { return Created; } - bool Execute(TColumnShard* owner, const TActorContext& ctx); + bool Execute(TColumnShard* owner, const TActorContext& ctx) const; + void Abort(TColumnShard* owner, const TString& reason, const TActorContext& ctx) const; }; class TWriteTasksQueue { private: bool WriteTasksOverloadCheckerScheduled = false; - THashMap> WriteTasks; - i64 PredWriteTasksSize = 0; + std::set WriteTasks; TColumnShard* Owner; public: diff --git a/ydb/core/tx/data_events/shard_writer.cpp b/ydb/core/tx/data_events/shard_writer.cpp index a31f48db69d4..18e98bb03790 100644 --- a/ydb/core/tx/data_events/shard_writer.cpp +++ b/ydb/core/tx/data_events/shard_writer.cpp @@ -58,6 +58,9 @@ namespace NKikimr::NEvWrite { void TShardWriter::SendWriteRequest() { auto ev = MakeHolder(NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE); DataForShard->Serialize(*ev, TableId, SchemaVersion); + if (Timeout) { + ev->Record.SetTimeoutSeconds(Timeout->Seconds()); + } SendToTablet(std::move(ev)); } From 2f5ac5c47ac99e618f341e5918fd427e2183bfb1 Mon Sep 17 00:00:00 2001 From: Semyon Date: Fri, 27 Jun 2025 16:16:03 +0300 Subject: [PATCH 40/62] fix portion accessor fetching for cleanup (#20252) --- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 27 +++++++++++++++++++ ydb/core/tx/columnshard/columnshard_impl.cpp | 2 +- .../engines/changes/abstract/abstract.h | 8 +++--- .../engines/changes/cleanup_portions.h | 4 +-- ydb/core/tx/columnshard/engines/changes/ttl.h | 2 +- .../engines/changes/with_appended.h | 4 +-- .../engines/column_engine_logs.cpp | 4 +-- .../columnshard/engines/ut/ut_logs_engine.cpp | 11 ++++++-- ydb/core/tx/columnshard/tables_manager.cpp | 1 + 9 files changed, 48 insertions(+), 15 deletions(-) diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 9d026a8910c7..ad2d09dd37c0 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -3910,5 +3910,32 @@ Y_UNIT_TEST_SUITE(KqpOlap) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); } } + + Y_UNIT_TEST(DropTable) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetOverridePeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetOverrideMaxReadStaleness(TDuration::Seconds(1)); + auto settings = TKikimrSettings().SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + TLocalHelper(kikimr).CreateTestOlapTable(); + WriteTestData(kikimr, "/Root/olapStore/olapTable", 0, 1000000, 20); + auto client = kikimr.GetTableClient(); + Tests::NCommon::TLoggerInit(kikimr).Initialize(); + + { + auto result = kikimr.GetQueryClient().ExecuteQuery("DROP TABLE `olapStore/olapTable`", NQuery::TTxControl::NoTx()).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + csController->WaitCleaning(TDuration::Seconds(5)); + + { + auto result = kikimr.GetQueryClient() + .ExecuteQuery("SELECT * FROM `olapStore/.sys/store_primary_index_portion_stats`", NQuery::TTxControl::NoTx()) + .GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + UNIT_ASSERT_EQUAL(result.GetResultSet(0).RowsCount(), 0); + } + } } } diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 4659caf6385e..38a47361f7f8 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -795,7 +795,7 @@ void TColumnShard::SetupCleanupPortions() { auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); NOlap::NDataFetcher::TRequestInput rInput( - changes->GetPortionsToDrop(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier()); + changes->GetPortionsToAccess(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier()); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching(std::move(rInput), std::make_shared( diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h index 916c198a2bbb..3681df049323 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.h @@ -285,8 +285,7 @@ class TColumnEngineChanges: public TMoveOnly { return DoBuildDataLock(); } - std::shared_ptr PortionsToAccess = - std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::GENERAL_COMPACTION); + std::vector PortionsToAccess; virtual void OnDataAccessorsInitialized(const TDataAccessorsInitializationContext& context) = 0; public: @@ -301,9 +300,8 @@ class TColumnEngineChanges: public TMoveOnly { ActivityFlag = flag; } - std::shared_ptr ExtractDataAccessorsRequest() { - AFL_VERIFY(!!PortionsToAccess); - return std::move(PortionsToAccess); + const std::vector& GetPortionsToAccess() const { + return PortionsToAccess; } const TPortionDataAccessor& GetPortionDataAccessor(const ui64 portionId) const { diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h index d3b34187ddd2..021d3db5e0e8 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.h @@ -64,12 +64,12 @@ class TCleanupPortionsColumnEngineChanges: public TColumnEngineChanges, void AddPortionToDrop(const TPortionInfo::TConstPtr& portion) { PortionsToDrop.emplace_back(portion); - PortionsToAccess->AddPortion(portion); + PortionsToAccess.emplace_back(portion); } void AddPortionToRemove(const TPortionInfo::TConstPtr& portion) { PortionsToRemove.AddPortion(portion); - PortionsToAccess->AddPortion(portion); + PortionsToAccess.emplace_back(portion); } virtual ui32 GetWritePortionsCount() const override { diff --git a/ydb/core/tx/columnshard/engines/changes/ttl.h b/ydb/core/tx/columnshard/engines/changes/ttl.h index 2406f866f717..edea05110702 100644 --- a/ydb/core/tx/columnshard/engines/changes/ttl.h +++ b/ydb/core/tx/columnshard/engines/changes/ttl.h @@ -109,7 +109,7 @@ class TTTLColumnEngineChanges: public TChangesWithAppend, public NColumnShard::T void AddPortionToEvict(const TPortionInfo::TConstPtr& info, TPortionEvictionFeatures&& features) { AFL_VERIFY(!info->HasRemoveSnapshot()); PortionsToEvict.emplace_back(info, std::move(features)); - PortionsToAccess->AddPortion(info); + PortionsToAccess.emplace_back(info); } std::vector GetPortionsInfo() const { diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.h b/ydb/core/tx/columnshard/engines/changes/with_appended.h index d160e091d0c0..26e6cd5bd089 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.h +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.h @@ -72,14 +72,14 @@ class TChangesWithAppend: public TColumnEngineChanges { void AddMovePortions(const std::vector>& portions) { PortionsToMove.AddPortions(portions); for (auto&& i : portions) { - PortionsToAccess->AddPortion(i); + PortionsToAccess.emplace_back(i); } } void AddPortionToRemove(const TPortionInfo::TConstPtr& info, const bool addIntoDataAccessRequest = true) { AFL_VERIFY(PortionsToRemove.AddPortion(info)); if (addIntoDataAccessRequest) { - PortionsToAccess->AddPortion(info); + PortionsToAccess.emplace_back(info); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index c05783156fa2..df373af492cd 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -297,10 +297,10 @@ std::shared_ptr TColumnEngineForLogs::Start } } AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartCleanup")("portions_count", CleanupPortions.size())( - "portions_prepared", changes->GetPortionsToDrop().size())("drop", portionsFromDrop)("skip", skipLocked)("portions_counter", portionsCount)( + "portions_prepared", changes->GetPortionsToAccess().size())("drop", portionsFromDrop)("skip", skipLocked)("portions_counter", portionsCount)( "chunks", chunksCount)("limit", limitExceeded)("max_portions", maxPortionsCount)("max_chunks", maxChunksCount); - if (changes->GetPortionsToDrop().empty()) { + if (changes->GetPortionsToAccess().empty()) { return nullptr; } diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 20efcd84bc31..b3d5aa8a95de 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -345,7 +346,10 @@ bool Compact(TColumnEngineForLogs& engine, TTestDbWrapper& db, TSnapshot snap, N // UNIT_ASSERT_VALUES_EQUAL(changes->SwitchedPortions.size(), expected.SrcPortions); changes->StartEmergency(); { - auto request = changes->ExtractDataAccessorsRequest(); + auto request = std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::GENERAL_COMPACTION); + for (const auto& portion : changes->GetPortionsToAccess()) { + request->AddPortion(portion); + } request->RegisterSubscriber( std::make_shared(changes, std::make_shared(engine.GetVersionedIndex()))); engine.FetchDataAccessors(request); @@ -435,7 +439,10 @@ bool Ttl(TColumnEngineForLogs& engine, TTestDbWrapper& db, const THashMapStartEmergency(); { - auto request = changes->ExtractDataAccessorsRequest(); + auto request = std::make_shared(NGeneralCache::TPortionsMetadataCachePolicy::EConsumer::GENERAL_COMPACTION); + for (const auto& portion : changes->GetPortionsToAccess()) { + request->AddPortion(portion); + } request->RegisterSubscriber( std::make_shared(changes, std::make_shared(engine.GetVersionedIndex()))); engine.FetchDataAccessors(request); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 4a53cb8f56e1..bdbf9e6858f2 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -443,6 +443,7 @@ bool TTablesManager::TryFinalizeDropPathOnComplete(const TInternalPathId pathId) } AFL_VERIFY(!GetPrimaryIndexSafe().HasDataInPathId(pathId)); AFL_VERIFY(MutablePrimaryIndex().ErasePathId(pathId)); + AFL_VERIFY(SchemeShardLocalToInternal.erase(itTable->second.GetPathId().GetSchemeShardLocalPathId())); Tables.erase(itTable); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("method", "TryFinalizeDropPathOnComplete")("path_id", pathId)("size", Tables.size()); return true; From 3e43f754c4fe8db0d1c26a43312b064223d18eb7 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 27 Jun 2025 16:20:44 +0300 Subject: [PATCH 41/62] blobs portion data into portion accessor (#20281) --- ydb/core/tx/columnshard/columnshard.cpp | 10 +- ydb/core/tx/columnshard/columnshard_impl.cpp | 9 +- ydb/core/tx/columnshard/columnshard_schema.h | 91 ++++++++--- .../counters/aggregation/table_stats.h | 5 - .../tx/columnshard/counters/engine_logs.cpp | 8 - ydb/core/tx/columnshard/counters/portions.cpp | 43 +++--- ydb/core/tx/columnshard/counters/portions.h | 143 +++++++++++++----- .../data_accessor/in_mem/manager.cpp | 2 + .../changes/abstract/remove_portions.cpp | 5 +- .../tx/columnshard/engines/column_engine.cpp | 5 +- .../tx/columnshard/engines/db_wrapper.cpp | 58 ++++--- ydb/core/tx/columnshard/engines/db_wrapper.h | 40 +++-- .../engines/portions/compacted.cpp | 8 +- .../columnshard/engines/portions/compacted.h | 4 +- .../engines/portions/constructor_accessor.cpp | 46 +++--- .../engines/portions/constructor_accessor.h | 56 +++---- .../engines/portions/constructor_meta.cpp | 14 +- .../engines/portions/constructor_meta.h | 22 +-- .../engines/portions/constructor_portion.h | 20 +-- .../engines/portions/data_accessor.cpp | 61 ++++---- .../engines/portions/data_accessor.h | 36 +++-- .../tx/columnshard/engines/portions/meta.cpp | 5 +- .../tx/columnshard/engines/portions/meta.h | 79 ++++++---- .../engines/portions/portion_info.cpp | 4 +- .../engines/portions/portion_info.h | 41 ++--- .../columnshard/engines/portions/written.cpp | 17 ++- .../tx/columnshard/engines/portions/written.h | 7 +- .../engines/protos/portion_info.proto | 4 + .../reader/plain_reader/iterator/source.cpp | 2 +- .../reader/simple_reader/iterator/source.cpp | 2 +- .../reader/simple_reader/iterator/source.h | 2 +- .../engines/reader/sys_view/chunks/chunks.cpp | 4 +- .../scheme/indexes/abstract/fetcher.cpp | 2 +- .../engines/storage/granule/granule.cpp | 6 +- .../engines/storage/granule/granule.h | 4 +- .../engines/storage/granule/stages.cpp | 23 +-- .../engines/storage/granule/stages.h | 53 ++++--- .../normalizer/abstract/abstract.h | 1 + .../normalizer/portion/broken_blobs.cpp | 2 +- .../portion/copy_blob_ids_to_v2.cpp | 115 ++++++++++++++ .../normalizer/portion/copy_blob_ids_to_v2.h | 47 ++++++ .../normalizer/portion/leaked_blobs.cpp | 38 ++--- .../normalizer/portion/leaked_blobs.h | 4 +- .../normalizer/portion/normalizer.cpp | 32 ++-- .../tx/columnshard/normalizer/portion/ya.make | 1 + .../tx/columnshard/ut_rw/ut_normalizer.cpp | 25 ++- 46 files changed, 750 insertions(+), 456 deletions(-) create mode 100644 ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.cpp create mode 100644 ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.h diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 9f2b694532c7..4d2c81d204a2 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -296,7 +296,7 @@ void TColumnShard::UpdateIndexCounters() { auto insertedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_INSERTED_PORTIONS, insertedStats.GetCount()); - counters->SetCounter(COUNTER_INSERTED_BLOBS, insertedStats.GetBlobs()); +// counters->SetCounter(COUNTER_INSERTED_BLOBS, insertedStats.GetBlobs()); counters->SetCounter(COUNTER_INSERTED_ROWS, insertedStats.GetRecordsCount()); counters->SetCounter(COUNTER_INSERTED_BYTES, insertedStats.GetBlobBytes()); counters->SetCounter(COUNTER_INSERTED_RAW_BYTES, insertedStats.GetRawBytes()); @@ -304,7 +304,7 @@ void TColumnShard::UpdateIndexCounters() { auto compactedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_COMPACTED_PORTIONS, compactedStats.GetCount()); - counters->SetCounter(COUNTER_COMPACTED_BLOBS, compactedStats.GetBlobs()); +// counters->SetCounter(COUNTER_COMPACTED_BLOBS, compactedStats.GetBlobs()); counters->SetCounter(COUNTER_COMPACTED_ROWS, compactedStats.GetRecordsCount()); counters->SetCounter(COUNTER_COMPACTED_BYTES, compactedStats.GetBlobBytes()); counters->SetCounter(COUNTER_COMPACTED_RAW_BYTES, compactedStats.GetRawBytes()); @@ -312,7 +312,7 @@ void TColumnShard::UpdateIndexCounters() { auto splitCompactedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_PORTIONS, splitCompactedStats.GetCount()); - counters->SetCounter(COUNTER_SPLIT_COMPACTED_BLOBS, splitCompactedStats.GetBlobs()); +// counters->SetCounter(COUNTER_SPLIT_COMPACTED_BLOBS, splitCompactedStats.GetBlobs()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_ROWS, splitCompactedStats.GetRecordsCount()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_BYTES, splitCompactedStats.GetBlobBytes()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_RAW_BYTES, splitCompactedStats.GetRawBytes()); @@ -320,7 +320,7 @@ void TColumnShard::UpdateIndexCounters() { auto inactiveStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_INACTIVE_PORTIONS, inactiveStats.GetCount()); - counters->SetCounter(COUNTER_INACTIVE_BLOBS, inactiveStats.GetBlobs()); +// counters->SetCounter(COUNTER_INACTIVE_BLOBS, inactiveStats.GetBlobs()); counters->SetCounter(COUNTER_INACTIVE_ROWS, inactiveStats.GetRecordsCount()); counters->SetCounter(COUNTER_INACTIVE_BYTES, inactiveStats.GetBlobBytes()); counters->SetCounter(COUNTER_INACTIVE_RAW_BYTES, inactiveStats.GetRawBytes()); @@ -328,7 +328,7 @@ void TColumnShard::UpdateIndexCounters() { auto evictedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_EVICTED_PORTIONS, evictedStats.GetCount()); - counters->SetCounter(COUNTER_EVICTED_BLOBS, evictedStats.GetBlobs()); +// counters->SetCounter(COUNTER_EVICTED_BLOBS, evictedStats.GetBlobs()); counters->SetCounter(COUNTER_EVICTED_ROWS, evictedStats.GetRecordsCount()); counters->SetCounter(COUNTER_EVICTED_BYTES, evictedStats.GetBlobBytes()); counters->SetCounter(COUNTER_EVICTED_RAW_BYTES, evictedStats.GetRawBytes()); diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 38a47361f7f8..dd3a7db0912c 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -1096,8 +1096,7 @@ class TPortionConstructorV2 { AFL_VERIFY(PortionInfo); AFL_VERIFY(Records)("portion_id", PortionInfo->GetPortionId())("path_id", PortionInfo->GetPathId()); AFL_VERIFY(Indexes)("portion_id", PortionInfo->GetPortionId())("path_id", PortionInfo->GetPathId()); - std::vector records = Records->BuildRecordsV1(); - return NOlap::TPortionAccessorConstructor::BuildForLoading(std::move(PortionInfo), std::move(records), std::move(*Indexes)); + return NOlap::TPortionAccessorConstructor::BuildForLoading(std::move(PortionInfo), Records->CreateBuildInfo(), std::move(*Indexes)); } }; @@ -1126,9 +1125,7 @@ class TAccessorsParsingTask: public NConveyor::ITask { TAccessorsParsingTask( const std::shared_ptr& callback, std::vector&& portions) : FetchCallback(callback) - , Portions(std::move(portions)) - { - + , Portions(std::move(portions)) { } }; @@ -1176,7 +1173,7 @@ class TTxAskPortionChunks: public TTransactionBase { } else { AFL_VERIFY(!rowset.EndOfSet())("path_id", i.first)("portion_id", p)( "debug", itPortionConstructor->second.GetPortionInfo()->DebugString(true)); - NOlap::TColumnChunkLoadContextV2 info(rowset); + NOlap::TColumnChunkLoadContextV2 info(rowset, selector); itPortionConstructor->second.SetRecords(std::move(info)); } } diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 6be107778adf..daeaf51ae76e 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -338,11 +338,12 @@ struct Schema : NIceDb::Schema { struct Metadata : Column<11, NScheme::NTypeIds::String> {}; // NKikimrTxColumnShard.TIndexColumnMeta struct Offset : Column<12, NScheme::NTypeIds::Uint32> {}; struct Size : Column<13, NScheme::NTypeIds::Uint32> {}; - struct PathId : Column<14, NScheme::NTypeIds::Uint64> {}; + struct PathId: Column<14, NScheme::NTypeIds::Uint64> {}; + struct BlobIdx: Column<15, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct IndexCounters : NIceDb::Schema::Table { @@ -412,9 +413,10 @@ struct Schema : NIceDb::Schema { struct RecordsCount: Column<8, NScheme::NTypeIds::Uint32> {}; struct RawBytes: Column<9, NScheme::NTypeIds::Uint64> {}; struct BlobData: Column<10, NScheme::NTypeIds::String> {}; + struct BlobIdx: Column<11, NScheme::NTypeIds::Uint32> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; struct SharedBlobIds: NIceDb::Schema::Table { @@ -575,9 +577,10 @@ struct Schema : NIceDb::Schema { struct PathId: Column<1, NScheme::NTypeIds::Uint64> {}; struct PortionId: Column<2, NScheme::NTypeIds::Uint64> {}; struct Metadata: Column<3, NScheme::NTypeIds::String> {}; + struct BlobIds: Column<4, NScheme::NTypeIds::String> {}; using TKey = TableKey; - using TColumns = TableColumns; + using TColumns = TableColumns; }; using TTables = SchemaTables< @@ -1003,13 +1006,26 @@ class TColumnChunkLoadContextV2 { YDB_READONLY_DEF(TInternalPathId, PathId); YDB_READONLY(ui64, PortionId, 0); YDB_READONLY_DEF(TString, MetadataProto); + YDB_READONLY_DEF(std::vector, BlobIds); public: + TPortionAddress GetPortionAddress() const { + return TPortionAddress(PathId, PortionId); + } + template - TColumnChunkLoadContextV2(const TSource& rowset) { + TColumnChunkLoadContextV2(const TSource& rowset, const NOlap::IBlobGroupSelector& dsGroupSelector) { PathId = TInternalPathId::FromRawValue(rowset.template GetValue()); PortionId = rowset.template GetValue(); MetadataProto = rowset.template GetValue(); + const TString blobIdsProto = rowset.template GetValue(); + + NKikimrTxColumnShard::TIndexPortionBlobsInfo blobsProto; + AFL_VERIFY(blobsProto.ParseFromArray(blobIdsProto.data(), blobIdsProto.size()))("event", "cannot parse blobs data as protobuf"); + for (auto&& i : blobsProto.GetBlobIds()) { + TLogoBlobID logoBlobId = TLogoBlobID::FromBinary(i.data()); + BlobIds.emplace_back(NOlap::TUnifiedBlobId(dsGroupSelector.GetGroup(logoBlobId), logoBlobId)); + } } TColumnChunkLoadContextV2(const TInternalPathId pathId, const ui64 portionId, const NKikimrTxColumnShard::TIndexPortionAccessor& proto) @@ -1018,7 +1034,20 @@ class TColumnChunkLoadContextV2 { , MetadataProto(proto.SerializeAsString()) { } - std::vector BuildRecordsV1() const { + class TBuildInfo { + private: + YDB_ACCESSOR_DEF(std::vector, BlobIds); + YDB_ACCESSOR_DEF(std::vector, Records); + + public: + TBuildInfo(std::vector&& blobIds, std::vector&& records) + : BlobIds(std::move(blobIds)) + , Records(std::move(records)) { + AFL_VERIFY(BlobIds.size()); + } + }; + + TBuildInfo CreateBuildInfo() { std::vector records; NKikimrTxColumnShard::TIndexPortionAccessor metaProto; AFL_VERIFY(metaProto.ParseFromArray(MetadataProto.data(), MetadataProto.size()))("event", "cannot parse metadata as protobuf"); @@ -1027,13 +1056,14 @@ class TColumnChunkLoadContextV2 { TBlobRangeLink16::BuildFromProto(i.GetBlobRangeLink()).DetachResult(), i.GetChunkMetadata()); records.emplace_back(std::move(result)); } - return records; + return TBuildInfo(std::move(BlobIds), std::move(records)); } }; class TIndexChunkLoadContext { private: - YDB_READONLY_DEF(std::optional, BlobRange); + YDB_READONLY_DEF(std::optional, BlobRangeLink16); + YDB_READONLY_DEF(std::optional, BlobRangeAddress); YDB_READONLY_DEF(std::optional, BlobData); YDB_READONLY_DEF(TInternalPathId, PathId); YDB_READONLY(ui64, PortionId, 0); @@ -1046,8 +1076,10 @@ class TIndexChunkLoadContext { } ui32 GetDataSize() const { - if (BlobRange) { - return BlobRange->GetSize(); + if (BlobRangeLink16) { + return BlobRangeLink16->GetSize(); + } else if (BlobRangeAddress) { + return BlobRangeAddress->GetSize(); } else { AFL_VERIFY(!!BlobData); return BlobData->size(); @@ -1055,21 +1087,33 @@ class TIndexChunkLoadContext { } TIndexChunk BuildIndexChunk(const TBlobRangeLink16::TLinkId blobLinkId) const { - AFL_VERIFY(BlobRange); - return TIndexChunk(Address.GetColumnId(), Address.GetChunkIdx(), RecordsCount, RawBytes, BlobRange->BuildLink(blobLinkId)); + AFL_VERIFY(BlobRangeAddress); + return TIndexChunk(Address.GetColumnId(), Address.GetChunkIdx(), RecordsCount, RawBytes, BlobRangeAddress->BuildLink(blobLinkId)); } TIndexChunk BuildIndexChunk() const { - AFL_VERIFY(BlobData); - return TIndexChunk(Address.GetColumnId(), Address.GetChunkIdx(), RecordsCount, RawBytes, *BlobData); + if (BlobRangeLink16) { + return TIndexChunk(Address.GetColumnId(), Address.GetChunkIdx(), RecordsCount, RawBytes, *BlobRangeLink16); + } else { + AFL_VERIFY(BlobData); + return TIndexChunk(Address.GetColumnId(), Address.GetChunkIdx(), RecordsCount, RawBytes, *BlobData); + } } - TIndexChunk BuildIndexChunk(const TPortionInfo& portionInfo) const { + TIndexChunk BuildIndexChunk(const std::vector& blobIds) const { if (BlobData) { return BuildIndexChunk(); + } else if (BlobRangeLink16) { + return BuildIndexChunk(); } else { - AFL_VERIFY(!!BlobRange); - return BuildIndexChunk(portionInfo.GetMeta().GetBlobIdxVerified(BlobRange->BlobId)); + AFL_VERIFY(BlobRangeAddress); + for (ui32 i = 0; i < blobIds.size(); ++i) { + if (blobIds[i] == BlobRangeAddress->BlobId) { + return BuildIndexChunk(i); + } + } + AFL_VERIFY(false); + return BuildIndexChunk(); } } @@ -1082,11 +1126,18 @@ class TIndexChunkLoadContext { , RawBytes(rowset.template GetValue()) { AFL_VERIFY(Address.GetColumnId())("event", "incorrect address")("address", Address.DebugString()); - if (rowset.template HaveValue()) { - TBlobRange& bRange = BlobRange.emplace(); + if (rowset.template HaveValue()) { + TBlobRangeLink16& bRange = BlobRangeLink16.emplace(); + bRange.BlobIdx = rowset.template GetValue(); + bRange.Offset = rowset.template GetValue(); + bRange.Size = rowset.template GetValue(); + AFL_VERIFY(bRange.Size)("event", "incorrect blob")("blob", bRange.ToString()); + } else if (rowset.template HaveValue()) { + TBlobRange& bRange = BlobRangeAddress.emplace(); TString strBlobId = rowset.template GetValue(); Y_ABORT_UNLESS(strBlobId.size() == sizeof(TLogoBlobID), "Size %" PRISZT " doesn't match TLogoBlobID", strBlobId.size()); TLogoBlobID logoBlobId((const ui64*)strBlobId.data()); + AFL_VERIFY(dsGroupSelector); bRange.BlobId = NOlap::TUnifiedBlobId(dsGroupSelector->GetGroup(logoBlobId), logoBlobId); bRange.Offset = rowset.template GetValue(); bRange.Size = rowset.template GetValue(); diff --git a/ydb/core/tx/columnshard/counters/aggregation/table_stats.h b/ydb/core/tx/columnshard/counters/aggregation/table_stats.h index e038bfbfa696..d4591ee22322 100644 --- a/ydb/core/tx/columnshard/counters/aggregation/table_stats.h +++ b/ydb/core/tx/columnshard/counters/aggregation/table_stats.h @@ -15,11 +15,6 @@ class TTableStatsBuilder { void FillPortionStats(::NKikimrTableStats::TTableStats& to, const NOlap::TSimplePortionsGroupInfo& from) const { to.SetRowCount(from.GetRecordsCount()); - for (const auto& [channel, bytes] : from.GetBytesByChannel()) { - auto item = to.AddChannels(); - item->SetChannel(channel); - item->SetDataSize(bytes); - } to.SetDataSize(from.GetBlobBytes()); } diff --git a/ydb/core/tx/columnshard/counters/engine_logs.cpp b/ydb/core/tx/columnshard/counters/engine_logs.cpp index 312e90f114ee..02f6a2c258b9 100644 --- a/ydb/core/tx/columnshard/counters/engine_logs.cpp +++ b/ydb/core/tx/columnshard/counters/engine_logs.cpp @@ -82,21 +82,13 @@ void TEngineLogsCounters::OnActualizationTask(const ui32 evictCount, const ui32 void TEngineLogsCounters::TPortionsInfoGuard::OnNewPortion(const std::shared_ptr& portion) const { const ui32 producedId = (ui32)portion->GetProduced(); - Y_ABORT_UNLESS(producedId < BlobGuards.size()); - for (auto&& blobId : portion->GetBlobIds()) { - BlobGuards[producedId]->Add(blobId.BlobSize(), blobId.BlobSize()); - } PortionRecordCountGuards[producedId]->Add(portion->GetRecordsCount(), 1); PortionSizeGuards[producedId]->Add(portion->GetTotalBlobBytes(), 1); } void TEngineLogsCounters::TPortionsInfoGuard::OnDropPortion(const std::shared_ptr& portion) const { const ui32 producedId = (ui32)portion->GetProduced(); - Y_ABORT_UNLESS(producedId < BlobGuards.size()); THashSet blobIds; - for (auto&& blobId : portion->GetBlobIds()) { - BlobGuards[producedId]->Sub(blobId.BlobSize(), blobId.BlobSize()); - } PortionRecordCountGuards[producedId]->Sub(portion->GetRecordsCount(), 1); PortionSizeGuards[producedId]->Sub(portion->GetTotalBlobBytes(), 1); } diff --git a/ydb/core/tx/columnshard/counters/portions.cpp b/ydb/core/tx/columnshard/counters/portions.cpp index 0d8c78db8c6b..35973d1f8ebd 100644 --- a/ydb/core/tx/columnshard/counters/portions.cpp +++ b/ydb/core/tx/columnshard/counters/portions.cpp @@ -1,4 +1,6 @@ #include "portions.h" + +#include #include namespace NKikimr::NColumnShard { @@ -21,36 +23,31 @@ void TPortionCategoryCounters::RemovePortion(const std::shared_ptr& p) { - AFL_VERIFY(p); - AddPortion(*p); +void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) { + BlobBytes.Sub(p.GetTotalBlobBytes()); + RawBytes.Sub(p.GetTotalRawBytes()); + Count.Sub(1); + RecordsCount.Sub(p.GetRecordsCount()); } + void TSimplePortionsGroupInfo::AddPortion(const TPortionInfo& p) { - Blobs += p.GetBlobIdsCount(); - BlobBytes += p.GetTotalBlobBytes(); - RawBytes += p.GetTotalRawBytes(); - Count += 1; - RecordsCount += p.GetRecordsCount(); + BlobBytes.Add(p.GetTotalBlobBytes()); + RawBytes.Add(p.GetTotalRawBytes()); + Count.Inc(); + RecordsCount.Add(p.GetRecordsCount()); +} + +void TFullPortionsGroupInfo::AddPortion(const TPortionDataAccessor& p) { + TBase::AddPortion(p.GetPortionInfo()); + Blobs.Add(p.GetBlobIdsCount()); for (const auto& blob : p.GetBlobIds()) { BytesByChannel[blob.Channel()] += blob.BlobSize(); } } -void TSimplePortionsGroupInfo::RemovePortion(const std::shared_ptr& p) { - AFL_VERIFY(p); - RemovePortion(*p); -} -void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) { - Blobs -= p.GetBlobIdsCount(); - BlobBytes -= p.GetTotalBlobBytes(); - RawBytes -= p.GetTotalRawBytes(); - Count -= 1; - RecordsCount -= p.GetRecordsCount(); - AFL_VERIFY(Blobs >= 0); - AFL_VERIFY(RawBytes >= 0); - AFL_VERIFY(BlobBytes >= 0); - AFL_VERIFY(Count >= 0); - AFL_VERIFY(RecordsCount >= 0); +void TFullPortionsGroupInfo::RemovePortion(const TPortionDataAccessor& p) { + TBase::RemovePortion(p.GetPortionInfo()); + Blobs.Sub(p.GetBlobIdsCount()); for (const auto& blob : p.GetBlobIds()) { auto findChannel = BytesByChannel.find(blob.Channel()); AFL_VERIFY(!findChannel.IsEnd())("blob", blob.ToStringLegacy()); diff --git a/ydb/core/tx/columnshard/counters/portions.h b/ydb/core/tx/columnshard/counters/portions.h index cfc06459a521..01167715cdf0 100644 --- a/ydb/core/tx/columnshard/counters/portions.h +++ b/ydb/core/tx/columnshard/counters/portions.h @@ -1,41 +1,56 @@ #pragma once +#include +#include #include #include #include -#include - #include #include namespace NKikimr::NOlap { class TPortionInfo; +class TPortionDataAccessor; class TSimplePortionsGroupInfo { private: using TCountByChannel = THashMap; - YDB_READONLY(i64, Blobs, 0); - YDB_READONLY(i64, BlobBytes, 0); - YDB_READONLY(i64, RawBytes, 0); - YDB_READONLY(i64, Count, 0); - YDB_READONLY(i64, RecordsCount, 0); - YDB_READONLY_DEF(TCountByChannel, BytesByChannel); + TPositiveControlInteger BlobBytes; + TPositiveControlInteger RawBytes; + TPositiveControlInteger Count; + TPositiveControlInteger RecordsCount; + +protected: + void Add(const TSimplePortionsGroupInfo& item) { + BlobBytes.Add(item.BlobBytes); + RawBytes.Add(item.RawBytes); + Count.Add(item.Count); + RecordsCount.Add(item.RecordsCount); + } public: + ui64 GetCount() const { + return Count.Val(); + } + + ui64 GetRecordsCount() const { + return RecordsCount.Val(); + } + + ui64 GetBlobBytes() const { + return BlobBytes.Val(); + } + + ui64 GetRawBytes() const { + return RawBytes.Val(); + } + NJson::TJsonValue SerializeToJson() const { NJson::TJsonValue result = NJson::JSON_MAP; - result.InsertValue("blobs", Blobs); - result.InsertValue("blob_bytes", BlobBytes); - result.InsertValue("raw_bytes", RawBytes); - result.InsertValue("count", Count); - result.InsertValue("records_count", RecordsCount); - { - NJson::TJsonValue bytesByChannel = NJson::JSON_MAP; - for (const auto& [channel, bytes] : BytesByChannel) { - bytesByChannel.InsertValue(ToString(channel), bytes); - } - result.InsertValue("bytes_by_channel", std::move(bytesByChannel)); - } + result.InsertValue("blob_bytes", BlobBytes.Val()); + result.InsertValue("raw_bytes", RawBytes.Val()); + result.InsertValue("count", Count.Val()); + result.InsertValue("records_count", RecordsCount.Val()); return result; } @@ -48,19 +63,12 @@ class TSimplePortionsGroupInfo { } TString DebugString() const { - return TStringBuilder() << "{blob_bytes=" << BlobBytes << ";raw_bytes=" << RawBytes << ";count=" << Count << ";records=" << RecordsCount - << "}"; + return TStringBuilder() << "{blob_bytes=" << BlobBytes.Val() << ";raw_bytes=" << RawBytes.Val() << ";count=" << Count.Val() + << ";records=" << RecordsCount.Val() << "}"; } TSimplePortionsGroupInfo& operator+=(const TSimplePortionsGroupInfo& item) { - Blobs += item.Blobs; - BlobBytes += item.BlobBytes; - RawBytes += item.RawBytes; - Count += item.Count; - RecordsCount += item.RecordsCount; - for (const auto& [channel, bytes] : item.BytesByChannel) { - BytesByChannel[channel] += bytes; - } + Add(item); return *this; } @@ -70,18 +78,81 @@ class TSimplePortionsGroupInfo { return result; } - void AddPortion(const std::shared_ptr& p); - void RemovePortion(const std::shared_ptr& p); + void AddPortion(const std::shared_ptr& p) { + AFL_VERIFY(p); + AddPortion(*p); + } void AddPortion(const TPortionInfo& p); + + void RemovePortion(const std::shared_ptr& p) { + AFL_VERIFY(p); + RemovePortion(*p); + } + void RemovePortion(const TPortionInfo& p); bool IsEmpty() const { - if (!Count) { + if (!Count.Val()) { + AFL_VERIFY(!BlobBytes.Val())("this", DebugString()); + AFL_VERIFY(!RawBytes.Val())("this", DebugString()); + AFL_VERIFY(!RecordsCount.Val())("this", DebugString()); + return true; + } + return false; + } +}; + +class TFullPortionsGroupInfo: public TSimplePortionsGroupInfo { +private: + using TBase = TSimplePortionsGroupInfo; + using TCountByChannel = THashMap; + TPositiveControlInteger Blobs; + YDB_READONLY_DEF(TCountByChannel, BytesByChannel); + + void Add(const TFullPortionsGroupInfo& item) { + TBase::Add(item); + Blobs.Add(item.Blobs); + for (const auto& [channel, bytes] : item.BytesByChannel) { + BytesByChannel[channel] += bytes; + } + } + +public: + NJson::TJsonValue SerializeToJson() const { + NJson::TJsonValue result = TBase::SerializeToJson(); + result.InsertValue("blobs", Blobs.Val()); + { + NJson::TJsonValue bytesByChannel = NJson::JSON_MAP; + for (const auto& [channel, bytes] : BytesByChannel) { + bytesByChannel.InsertValue(ToString(channel), bytes); + } + result.InsertValue("bytes_by_channel", std::move(bytesByChannel)); + } + return result; + } + + TString DebugString() const { + return TBase::DebugString(); + } + + TFullPortionsGroupInfo& operator+=(const TFullPortionsGroupInfo& item) { + Add(item); + return *this; + } + + TFullPortionsGroupInfo operator+(const TFullPortionsGroupInfo& item) const { + TFullPortionsGroupInfo result = *this; + result += item; + return result; + } + + void AddPortion(const TPortionDataAccessor& p); + void RemovePortion(const TPortionDataAccessor& p); + + bool IsEmpty() const { + if (TBase::IsEmpty()) { AFL_VERIFY(!Blobs)("this", DebugString()); - AFL_VERIFY(!BlobBytes)("this", DebugString()); - AFL_VERIFY(!RawBytes)("this", DebugString()); - AFL_VERIFY(!RecordsCount)("this", DebugString()); AFL_VERIFY(BytesByChannel.empty())("this", DebugString()); return true; } diff --git a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp index dde5c6fd48dd..4436a9ae79d3 100644 --- a/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp +++ b/ydb/core/tx/columnshard/data_accessor/in_mem/manager.cpp @@ -9,6 +9,8 @@ std::shared_ptr TManager::DoBuildLoader( const TVersionedIndex& versionedIndex, TGranuleMeta* granule, const std::shared_ptr& dsGroupSelector) { auto result = std::make_shared("granule"); auto portionsLoadContext = std::make_shared(); + result->AddChildren( + std::make_shared("start", &versionedIndex, granule, dsGroupSelector, portionsLoadContext)); result->AddChildren( std::make_shared("columns", &versionedIndex, granule, dsGroupSelector, portionsLoadContext)); result->AddChildren( diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/remove_portions.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/remove_portions.cpp index 13b89ab981f7..a5589e35e00b 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/remove_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/remove_portions.cpp @@ -36,10 +36,7 @@ void TRemovePortionsChange::DoApplyOnComplete( self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_PORTIONS_DEACTIVATED, Portions.size()); for (auto& [_, portionInfo] : Portions) { - self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_BLOBS_DEACTIVATED, portionInfo->GetBlobIdsCount()); - for (auto& blobId : portionInfo->GetBlobIds()) { - self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_BYTES_DEACTIVATED, blobId.BlobSize()); - } + self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_BYTES_DEACTIVATED, portionInfo->GetTotalBlobBytes()); self->Counters.GetTabletCounters()->IncCounter(NColumnShard::COUNTER_RAW_BYTES_DEACTIVATED, portionInfo->GetTotalRawBytes()); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine.cpp b/ydb/core/tx/columnshard/engines/column_engine.cpp index bd6cf4925a4e..5f769893e1df 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine.cpp @@ -43,10 +43,7 @@ TSelectInfo::TStats TSelectInfo::Stats() const { THashSet uniqBlob; for (auto& portionInfo : Portions) { out.Rows += portionInfo->GetRecordsCount(); - for (auto& blobId : portionInfo->GetBlobIds()) { - out.Bytes += blobId.BlobSize(); - } - out.Blobs += portionInfo->GetBlobIdsCount(); + out.Bytes += portionInfo->GetTotalBlobBytes(); } return out; } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 008f17e41324..8a337f3884bc 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -9,7 +9,8 @@ namespace NKikimr::NOlap { -void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) { +void TDbWrapper::WriteColumn( + const TPortionDataAccessor& acc, const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) { if (!AppDataVerified().ColumnShardConfig.GetColumnChunksV1Usage() && !AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { return; } @@ -25,9 +26,8 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe } if (AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { if (row.GetChunkIdx() == 0 && row.GetColumnId() == firstPKColumnId) { - *rowProto.MutablePortionMeta() = - portion.GetMeta().SerializeToProto(portion.GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED - : NPortion::EProduced::INSERTED); + *rowProto.MutablePortionMeta() = portion.GetMeta().SerializeToProto(acc.GetBlobIds(), + portion.GetPortionType() == EPortionType::Compacted ? NPortion::EProduced::SPLIT_COMPACTED : NPortion::EProduced::INSERTED); } using IndexColumns = NColumnShard::Schema::IndexColumns; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); @@ -35,16 +35,37 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe .Key(0, 0, row.ColumnId, 1, 1, portion.GetPortionId(), row.Chunk) .Update(NIceDb::TUpdate(removeSnapshot ? removeSnapshot->GetPlanStep() : 0), NIceDb::TUpdate(removeSnapshot ? removeSnapshot->GetTxId() : 0), - NIceDb::TUpdate(portion.GetBlobId(row.GetBlobRange().GetBlobIdxVerified()).SerializeBinary()), + NIceDb::TUpdate(acc.GetBlobId(row.GetBlobRange().GetBlobIdxVerified()).SerializeBinary()), + NIceDb::TUpdate(row.GetBlobRange().GetBlobIdxVerified()), NIceDb::TUpdate(rowProto.SerializeAsString()), NIceDb::TUpdate(row.BlobRange.Offset), NIceDb::TUpdate(row.BlobRange.Size), NIceDb::TUpdate(portion.GetPathId().GetRawValue())); } } -void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { +void TDbWrapper::WritePortion(const std::vector& blobIds, const NOlap::TPortionInfo& portion) { NIceDb::TNiceDb db(Database); - portion.SaveMetaToDatabase(db); + portion.SaveMetaToDatabase(blobIds, db); +} + +void TDbWrapper::CommitPortion(const NOlap::TPortionInfo& portion, const TSnapshot& commitSnapshot) { + NIceDb::TNiceDb db(Database); + using IndexPortions = NColumnShard::Schema::IndexPortions; + if (portion.HasRemoveSnapshot()) { + db.Table() + .Key(portion.GetPathId().GetRawValue(), portion.GetPortionId()) + .Update( + NIceDb::TUpdate(commitSnapshot.GetPlanStep()), + NIceDb::TUpdate(commitSnapshot.GetTxId()), + NIceDb::TUpdate(portion.GetRemoveSnapshotVerified().GetPlanStep()), + NIceDb::TUpdate(portion.GetRemoveSnapshotVerified().GetTxId()) + ); + } else { + db.Table() + .Key(portion.GetPathId().GetRawValue(), portion.GetPortionId()) + .Update(NIceDb::TUpdate(commitSnapshot.GetPlanStep()), + NIceDb::TUpdate(commitSnapshot.GetTxId())); + } } void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { @@ -61,10 +82,7 @@ void TDbWrapper::EraseColumn(const NOlap::TPortionInfo& portion, const TColumnRe } if (AppDataVerified().ColumnShardConfig.GetColumnChunksV0Usage()) { using IndexColumns = NColumnShard::Schema::IndexColumns; - db.Table() - .Key(0, 0, row.ColumnId, 1, 1, - portion.GetPortionId(), row.Chunk) - .Delete(); + db.Table().Key(0, 0, row.ColumnId, 1, 1, portion.GetPortionId(), row.Chunk).Delete(); } } @@ -77,7 +95,8 @@ bool TDbWrapper::LoadColumns(const std::optional pathId, const } while (!rowset.EndOfSet()) { - NOlap::TColumnChunkLoadContextV2 chunkLoadContext(rowset); + AFL_VERIFY(DsGroupSelector); + NOlap::TColumnChunkLoadContextV2 chunkLoadContext(rowset, *DsGroupSelector); callback(std::move(chunkLoadContext)); if (!rowset.Next()) { @@ -149,16 +168,17 @@ bool TDbWrapper::LoadPortions(const std::optional pathId, } } -void TDbWrapper::WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) { +void TDbWrapper::WriteIndex(const TPortionDataAccessor& acc, const TPortionInfo& portion, const TIndexChunk& row) { using IndexIndexes = NColumnShard::Schema::IndexIndexes; NIceDb::TNiceDb db(Database); if (auto bRange = row.GetBlobRangeOptional()) { AFL_VERIFY(bRange->IsValid()); db.Table() .Key(portion.GetPathId().GetRawValue(), portion.GetPortionId(), row.GetIndexId(), row.GetChunkIdx()) - .Update(NIceDb::TUpdate(portion.GetBlobId(bRange->GetBlobIdxVerified()).SerializeBinary()), - NIceDb::TUpdate(bRange->Offset), NIceDb::TUpdate(row.GetDataSize()), - NIceDb::TUpdate(row.GetRecordsCount()), NIceDb::TUpdate(row.GetRawBytes())); + .Update(NIceDb::TUpdate(acc.GetBlobId(bRange->GetBlobIdxVerified()).SerializeBinary()), + NIceDb::TUpdate(bRange->GetBlobIdxVerified()), NIceDb::TUpdate(bRange->Offset), + NIceDb::TUpdate(row.GetDataSize()), NIceDb::TUpdate(row.GetRecordsCount()), + NIceDb::TUpdate(row.GetRawBytes())); } else if (auto bData = row.GetBlobDataOptional()) { db.Table() .Key(portion.GetPathId().GetRawValue(), portion.GetPortionId(), row.GetIndexId(), row.GetChunkIdx()) @@ -238,12 +258,14 @@ TConclusion() .Key(portion.GetPathId().GetRawValue(), portion.GetPortionId()) - .Update(NIceDb::TUpdate(proto.SerializeAsString())); + .Update(NIceDb::TUpdate(proto.SerializeAsString())) + .Update(NIceDb::TUpdate(protoBlobs.SerializeAsString())); } } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index d1848317305f..4be6d20ec6b5 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -1,9 +1,11 @@ #pragma once #include "defs.h" + #include "scheme/versions/versioned_index.h" + #include -#include #include +#include namespace NKikimrTxColumnShard { class TIndexPortionMeta; @@ -35,19 +37,22 @@ class IDbWrapper { return *result; } - virtual void WriteColumns(const NOlap::TPortionInfo& portion, const NKikimrTxColumnShard::TIndexPortionAccessor& proto) = 0; + virtual void WriteColumns(const NOlap::TPortionInfo& portion, const NKikimrTxColumnShard::TIndexPortionAccessor& proto, + const NKikimrTxColumnShard::TIndexPortionBlobsInfo& protoBlobs) = 0; - virtual void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0; + virtual void WriteColumn( + const TPortionDataAccessor& acc, const TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) = 0; virtual void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) = 0; virtual bool LoadColumns(const std::optional pathId, const std::function& callback) = 0; - virtual void WritePortion(const NOlap::TPortionInfo& portion) = 0; + virtual void WritePortion(const std::vector& blobIds, const NOlap::TPortionInfo& portion) = 0; + virtual void CommitPortion(const NOlap::TPortionInfo& portion, const TSnapshot& commitSnapshot) = 0; virtual void ErasePortion(const NOlap::TPortionInfo& portion) = 0; virtual bool LoadPortions(const std::optional pathId, const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& callback) = 0; - virtual void WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) = 0; + virtual void WriteIndex(const TPortionDataAccessor& acc, const TPortionInfo& portion, const TIndexChunk& row) = 0; virtual void EraseIndex(const TPortionInfo& portion, const TIndexChunk& row) = 0; virtual bool LoadIndexes(const std::optional pathId, const std::function& callback) = 0; @@ -57,23 +62,28 @@ class IDbWrapper { virtual TConclusion>> LoadGranulesShardingInfo() = 0; }; -class TDbWrapper : public IDbWrapper { +class TDbWrapper: public IDbWrapper { public: TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector) : Database(db) - , DsGroupSelector(dsGroupSelector) - {} + , DsGroupSelector(dsGroupSelector) { + } - void WritePortion(const NOlap::TPortionInfo& portion) override; + void WritePortion(const std::vector& blobIds, const NOlap::TPortionInfo& portion) override; + void CommitPortion(const NOlap::TPortionInfo& portion, const TSnapshot& commitSnapshot) override; void ErasePortion(const NOlap::TPortionInfo& portion) override; - bool LoadPortions(const std::optional pathId, const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& callback) override; - - void WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) override; - void WriteColumns(const NOlap::TPortionInfo& portion, const NKikimrTxColumnShard::TIndexPortionAccessor& proto) override; + bool LoadPortions(const std::optional pathId, + const std::function&&, const NKikimrTxColumnShard::TIndexPortionMeta&)>& callback) + override; + + void WriteColumn( + const TPortionDataAccessor& acc, const NOlap::TPortionInfo& portion, const TColumnRecord& row, const ui32 firstPKColumnId) override; + void WriteColumns(const NOlap::TPortionInfo& portion, const NKikimrTxColumnShard::TIndexPortionAccessor& proto, + const NKikimrTxColumnShard::TIndexPortionBlobsInfo& protoBlobs) override; void EraseColumn(const NOlap::TPortionInfo& portion, const TColumnRecord& row) override; bool LoadColumns(const std::optional pathId, const std::function& callback) override; - virtual void WriteIndex(const TPortionInfo& portion, const TIndexChunk& row) override; + virtual void WriteIndex(const TPortionDataAccessor& acc, const TPortionInfo& portion, const TIndexChunk& row) override; virtual void EraseIndex(const TPortionInfo& portion, const TIndexChunk& row) override; virtual bool LoadIndexes(const std::optional pathId, const std::function& callback) override; @@ -92,4 +102,4 @@ class TDbWrapper : public IDbWrapper { const IBlobGroupSelector* DsGroupSelector; }; -} +} // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/compacted.cpp b/ydb/core/tx/columnshard/engines/portions/compacted.cpp index 3464784009c2..4abf00c772b8 100644 --- a/ydb/core/tx/columnshard/engines/portions/compacted.cpp +++ b/ydb/core/tx/columnshard/engines/portions/compacted.cpp @@ -6,8 +6,8 @@ namespace NKikimr::NOlap { -void TCompactedPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const { - auto metaProto = GetMeta().SerializeToProto(NPortion::EProduced::SPLIT_COMPACTED); +void TCompactedPortionInfo::DoSaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const { + auto metaProto = GetMeta().SerializeToProto(blobIds, NPortion::EProduced::SPLIT_COMPACTED); using IndexPortions = NColumnShard::Schema::IndexPortions; const auto removeSnapshot = GetRemoveSnapshotOptional(); db.Table() @@ -30,8 +30,8 @@ const TSnapshot& TCompactedPortionInfo::RecordSnapshotMax(const std::optional TCompactedPortionInfo::BuildConstructor(const bool withMetadata, const bool withMetadataBlobs) const { - return std::make_unique(*this, withMetadata, withMetadataBlobs); +std::unique_ptr TCompactedPortionInfo::BuildConstructor(const bool withMetadata) const { + return std::make_unique(*this, withMetadata); } NSplitter::TEntityGroups TCompactedPortionInfo::GetEntityGroupsByStorageId( diff --git a/ydb/core/tx/columnshard/engines/portions/compacted.h b/ydb/core/tx/columnshard/engines/portions/compacted.h index 65e3671477c6..d6ac20c84e44 100644 --- a/ydb/core/tx/columnshard/engines/portions/compacted.h +++ b/ydb/core/tx/columnshard/engines/portions/compacted.h @@ -7,7 +7,7 @@ class TCompactedPortionInfo: public TPortionInfo { private: using TBase = TPortionInfo; friend class TPortionInfoConstructor; - virtual void DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const override; + virtual void DoSaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const override; virtual bool DoIsVisible(const TSnapshot& snapshot, const bool /*checkCommitSnapshot*/) const override { return RecordSnapshotMin(std::nullopt) <= snapshot; @@ -37,7 +37,7 @@ class TCompactedPortionInfo: public TPortionInfo { virtual const TString& GetColumnStorageId(const ui32 columnId, const TIndexInfo& indexInfo) const override; virtual const TString& GetEntityStorageId(const ui32 columnId, const TIndexInfo& indexInfo) const override; virtual const TString& GetIndexStorageId(const ui32 indexId, const TIndexInfo& indexInfo) const override; - virtual std::unique_ptr BuildConstructor(const bool withMetadata, const bool withMetadataBlobs) const override; + virtual std::unique_ptr BuildConstructor(const bool withMetadata) const override; virtual const TSnapshot& RecordSnapshotMin(const std::optional& /*snapshotDefault*/) const override; virtual const TSnapshot& RecordSnapshotMax(const std::optional& /*snapshotDefault*/) const override; }; diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_accessor.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_accessor.cpp index 41786d29a7b2..f23c9ee88438 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_accessor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor_accessor.cpp @@ -14,18 +14,18 @@ void TPortionAccessorConstructor::ChunksValidation() const { } else { std::set blobIdxs; for (auto&& i : Records) { - TBlobRange::Validate(PortionInfo->MetaConstructor.BlobIds, i.GetBlobRange()).Validate(); + TBlobRange::Validate(GetBlobIds(), i.GetBlobRange()).Validate(); blobIdxs.emplace(i.GetBlobRange().GetBlobIdxVerified()); } for (auto&& i : Indexes) { if (i.HasBlobRange()) { - TBlobRange::Validate(PortionInfo->MetaConstructor.BlobIds, i.GetBlobRangeVerified()).Validate(); + TBlobRange::Validate(GetBlobIds(), i.GetBlobRangeVerified()).Validate(); blobIdxs.emplace(i.GetBlobRangeVerified().GetBlobIdxVerified()); } } - if (PortionInfo->MetaConstructor.BlobIds.size()) { - AFL_VERIFY(PortionInfo->MetaConstructor.BlobIds.size() == blobIdxs.size()); - AFL_VERIFY(PortionInfo->MetaConstructor.BlobIds.size() == *blobIdxs.rbegin() + 1); + if (GetBlobIdsCount()) { + AFL_VERIFY(GetBlobIdsCount() == blobIdxs.size()); + AFL_VERIFY(GetBlobIdsCount() == *blobIdxs.rbegin() + 1); } else { AFL_VERIFY(blobIdxs.empty()); } @@ -90,38 +90,40 @@ TPortionDataAccessor TPortionAccessorConstructor::Build(const bool needChunksNor AFL_VERIFY(itBlobIdx == BlobIdxs.end()); } else { for (auto&& i : Records) { - AFL_VERIFY(i.BlobRange.GetBlobIdxVerified() < PortionInfo->MetaConstructor.BlobIds.size()); + AFL_VERIFY(i.BlobRange.GetBlobIdxVerified() < GetBlobIdsCount()); } for (auto&& i : Indexes) { if (auto* blobId = i.GetBlobRangeOptional()) { - AFL_VERIFY(blobId->GetBlobIdxVerified() < PortionInfo->MetaConstructor.BlobIds.size()); + AFL_VERIFY(blobId->GetBlobIdxVerified() < GetBlobIdsCount()); } } } ChunksValidation(); - - return TPortionDataAccessor(result, std::move(Records), std::move(Indexes), false); + return TPortionDataAccessor(result, ExtractBlobIds(), std::move(Records), std::move(Indexes), false); } -void TPortionAccessorConstructor::LoadRecord(TColumnChunkLoadContextV1&& loadContext) { - AFL_VERIFY(loadContext.GetBlobRange().GetBlobIdxVerified() < PortionInfo->MetaConstructor.BlobIds.size()); - AFL_VERIFY(loadContext.GetBlobRange().CheckBlob(PortionInfo->MetaConstructor.BlobIds[loadContext.GetBlobRange().GetBlobIdxVerified()]))( - "blobs", JoinSeq(",", PortionInfo->MetaConstructor.BlobIds))("range", loadContext.GetBlobRange().ToString()); - TColumnRecord rec(loadContext); - Records.push_back(std::move(rec)); +void TPortionAccessorConstructor::AddBuildInfo(TColumnChunkLoadContextV2::TBuildInfo&& buildInfo) { + AFL_VERIFY(BlobIds.empty()); + BlobIds = buildInfo.DetachBlobIds(); + for (auto&& rec : buildInfo.DetachRecords()) { + AFL_VERIFY(rec.GetBlobRange().GetBlobIdxVerified() < GetBlobIdsCount()); + AFL_VERIFY(rec.GetBlobRange().CheckBlob(GetBlobId(rec.GetBlobRange().GetBlobIdxVerified())))( + "blobs", JoinSeq(",", GetBlobIds()))("range", rec.GetBlobRange().ToString()); + Records.push_back(std::move(rec)); + } } void TPortionAccessorConstructor::LoadIndex(TIndexChunkLoadContext&& loadContext) { - if (loadContext.GetBlobRange()) { - const TBlobRangeLink16::TLinkId linkBlobId = PortionInfo->GetMeta().GetBlobIdxVerified(loadContext.GetBlobRange()->GetBlobId()); + if (loadContext.GetBlobRangeAddress()) { + const TBlobRangeLink16::TLinkId linkBlobId = GetBlobIdxVerified(loadContext.GetBlobRangeAddress()->GetBlobId()); AddIndex(loadContext.BuildIndexChunk(linkBlobId)); } else { AddIndex(loadContext.BuildIndexChunk()); } } -TPortionDataAccessor TPortionAccessorConstructor::BuildForLoading( - const TPortionInfo::TConstPtr& portion, std::vector&& records, std::vector&& indexes) { +TPortionDataAccessor TPortionAccessorConstructor::BuildForLoading(const TPortionInfo::TConstPtr& portion, + TColumnChunkLoadContextV2::TBuildInfo&& records, std::vector&& indexes) { AFL_VERIFY(portion); std::vector recordChunks; { @@ -129,7 +131,7 @@ TPortionDataAccessor TPortionAccessorConstructor::BuildForLoading( return l.GetAddress() < r.GetAddress(); }; bool needSort = false; - for (auto&& i : records) { + for (auto&& i : records.GetRecords()) { TColumnRecord chunk(i); if (recordChunks.size() && !pred(recordChunks.back(), chunk)) { needSort = true; @@ -148,7 +150,7 @@ TPortionDataAccessor TPortionAccessorConstructor::BuildForLoading( }; bool needSort = false; for (auto&& i : indexes) { - auto chunk = i.BuildIndexChunk(*portion); + auto chunk = i.BuildIndexChunk(records.GetBlobIds()); if (indexChunks.size() && !pred(indexChunks.back(), chunk)) { needSort = true; } @@ -158,7 +160,7 @@ TPortionDataAccessor TPortionAccessorConstructor::BuildForLoading( std::sort(indexChunks.begin(), indexChunks.end(), pred); } } - return TPortionDataAccessor(portion, std::move(recordChunks), std::move(indexChunks), true); + return TPortionDataAccessor(portion, records.DetachBlobIds(), std::move(recordChunks), std::move(indexChunks), true); } } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_accessor.h b/ydb/core/tx/columnshard/engines/portions/constructor_accessor.h index bfff344b4a11..7230f0425b3f 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor_accessor.h @@ -4,12 +4,14 @@ #include "data_accessor.h" #include "index_chunk.h" +#include #include + #include namespace NKikimr::NOlap { -class TPortionAccessorConstructor { +class TPortionAccessorConstructor: public TPortionMetaBase { private: bool Constructed = false; std::unique_ptr PortionInfo; @@ -35,18 +37,23 @@ class TPortionAccessorConstructor { bool NeedBlobIdxsSort = false; TPortionAccessorConstructor(TPortionDataAccessor&& accessor) - : PortionInfo(accessor.GetPortionInfo().BuildConstructor(true, true)) { + : PortionInfo(accessor.GetPortionInfo().BuildConstructor(true)) { + BlobIds = accessor.ExtractBlobIds(); Indexes = accessor.ExtractIndexes(); Records = accessor.ExtractRecords(); } TPortionAccessorConstructor( const TPortionDataAccessor& accessor, const bool withBlobs, const bool withMetadata, const bool withMetadataBlobs) - : PortionInfo(accessor.GetPortionInfo().BuildConstructor(withMetadata, withMetadataBlobs)) { + : PortionInfo(accessor.GetPortionInfo().BuildConstructor(withMetadata)) { + if (!withMetadata) { + AFL_VERIFY(!withMetadataBlobs); + } if (withBlobs) { AFL_VERIFY(withMetadataBlobs && withMetadata); Indexes = accessor.GetIndexesVerified(); Records = accessor.GetRecordsVerified(); + BlobIds = accessor.GetBlobIds(); } } @@ -103,7 +110,7 @@ class TPortionAccessorConstructor { recordsCountCurrent = 0; } else { AFL_VERIFY(i.GetChunkIdx() == chunkIdx + 1)("chunkIdx", chunkIdx)("i.GetChunkIdx()", i.GetChunkIdx())("entity", entityId)( - "details", debugString()); + "details", debugString()); chunkIdx = i.GetChunkIdx(); } recordsCountCurrent += GetRecordsCount(i); @@ -138,10 +145,21 @@ class TPortionAccessorConstructor { } public: - TPortionAccessorConstructor(std::unique_ptr&& portionInfo) - : PortionInfo(std::move(portionInfo)) - { + TBlobRangeLink16::TLinkId RegisterBlobId(const TUnifiedBlobId& blobId) { + AFL_VERIFY(blobId.IsValid()); + TBlobRangeLink16::TLinkId idx = 0; + for (auto&& i : BlobIds) { + if (i == blobId) { + return idx; + } + ++idx; + } + BlobIds.emplace_back(blobId); + return idx; + } + TPortionAccessorConstructor(std::unique_ptr&& portionInfo) + : PortionInfo(std::move(portionInfo)) { } ui64 GetTotalBlobsSize() const { @@ -188,11 +206,11 @@ class TPortionAccessorConstructor { } static TPortionAccessorConstructor BuildForRewriteBlobs(const TPortionInfo& portion) { - return TPortionAccessorConstructor(portion.BuildConstructor(true, false)); + return TPortionAccessorConstructor(portion.BuildConstructor(true)); } static TPortionDataAccessor BuildForLoading( - const TPortionInfo::TConstPtr& portion, std::vector&& records, std::vector&& indexes); + const TPortionInfo::TConstPtr& portion, TColumnChunkLoadContextV2::TBuildInfo&& records, std::vector&& indexes); const std::vector& GetRecords() const { return Records; @@ -242,26 +260,10 @@ class TPortionAccessorConstructor { TPortionDataAccessor Build(const bool needChunksNormalization); - TBlobRangeLink16::TLinkId RegisterBlobId(const TUnifiedBlobId& blobId) { - return PortionInfo->MetaConstructor.RegisterBlobId(blobId); - } - - const TBlobRange RestoreBlobRange(const TBlobRangeLink16& linkRange) const { - return PortionInfo->MetaConstructor.RestoreBlobRange(linkRange); - } - - const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const { - return PortionInfo->MetaConstructor.GetBlobId(linkId); - } - - ui32 GetBlobIdsCount() const { - return PortionInfo->MetaConstructor.GetBlobIdsCount(); - } - TPortionAccessorConstructor(TPortionAccessorConstructor&&) noexcept = default; TPortionAccessorConstructor& operator=(TPortionAccessorConstructor&&) noexcept = default; - void LoadRecord(TColumnChunkLoadContextV1&& loadContext); + void AddBuildInfo(TColumnChunkLoadContextV2::TBuildInfo&& buildInfo); void LoadIndex(TIndexChunkLoadContext&& loadContext); const TColumnRecord& AppendOneChunkColumn(TColumnRecord&& record) { @@ -285,7 +287,7 @@ class TPortionAccessorConstructor { } bool HaveBlobsData() { - return PortionInfo->HaveBlobsData() || Records.size() || Indexes.size(); + return GetBlobIdsCount() || Records.size() || Indexes.size(); } void ClearRecords() { diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp index 9da7c73f17c0..9142fb0ea81f 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.cpp @@ -29,16 +29,13 @@ void TPortionMetaConstructor::FillMetaInfo(const NArrow::TFirstLastSpecialKeys& } } -TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta, const bool withBlobs) { +TPortionMetaConstructor::TPortionMetaConstructor(const TPortionMeta& meta) { FirstAndLastPK = NArrow::TFirstLastSpecialKeys(meta.IndexKeyStart(), meta.IndexKeyEnd(), meta.IndexKeyStart().GetSchema()); RecordSnapshotMin = meta.RecordSnapshotMin; RecordSnapshotMax = meta.RecordSnapshotMax; CompactionLevel = meta.GetCompactionLevel(); DeletionsCount = meta.GetDeletionsCount(); TierName = meta.GetTierNameOptional(); - if (withBlobs) { - BlobIds = meta.BlobIds; - } } TPortionMeta TPortionMetaConstructor::Build() { @@ -57,9 +54,6 @@ TPortionMeta TPortionMetaConstructor::Build() { if (TierName) { result.TierName = *TierName; } - TBase::FullValidation(); - result.BlobIds = BlobIds; - result.BlobIds.shrink_to_fit(); result.CompactionLevel = *TValidator::CheckNotNull(CompactionLevel); result.DeletionsCount = *TValidator::CheckNotNull(DeletionsCount); @@ -75,7 +69,7 @@ TPortionMeta TPortionMetaConstructor::Build() { } bool TPortionMetaConstructor::LoadMetadata( - const NKikimrTxColumnShard::TIndexPortionMeta& portionMeta, const TIndexInfo& indexInfo, const IBlobGroupSelector& groupSelector) { + const NKikimrTxColumnShard::TIndexPortionMeta& portionMeta, const TIndexInfo& indexInfo, const IBlobGroupSelector& /*groupSelector*/) { if (portionMeta.GetTierName()) { TierName = portionMeta.GetTierName(); } @@ -84,10 +78,6 @@ bool TPortionMetaConstructor::LoadMetadata( } else { DeletionsCount = 0; } - for (auto&& i : portionMeta.GetBlobIds()) { - TLogoBlobID logo = TLogoBlobID::FromBinary(i); - BlobIds.emplace_back(TUnifiedBlobId(groupSelector.GetGroup(logo), logo)); - } CompactionLevel = portionMeta.GetCompactionLevel(); RecordsCount = TValidator::CheckNotNull(portionMeta.GetRecordsCount()); ColumnRawBytes = TValidator::CheckNotNull(portionMeta.GetColumnRawBytes()); diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h index 9f21bdfe0988..5e1432168d91 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_meta.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor_meta.h @@ -10,9 +10,8 @@ namespace NKikimr::NOlap { class TPortionInfoConstructor; struct TIndexInfo; -class TPortionMetaConstructor: public TPortionMetaBase { +class TPortionMetaConstructor { private: - using TBase = TPortionMetaBase; std::optional FirstAndLastPK; std::optional TierName; std::optional RecordSnapshotMin; @@ -34,7 +33,7 @@ class TPortionMetaConstructor: public TPortionMetaBase { public: TPortionMetaConstructor() = default; - TPortionMetaConstructor(const TPortionMeta& meta, const bool withBlobs); + TPortionMetaConstructor(const TPortionMeta& meta); const NArrow::TFirstLastSpecialKeys& GetFirstAndLastPK() const { AFL_VERIFY(FirstAndLastPK); @@ -47,23 +46,6 @@ class TPortionMetaConstructor: public TPortionMetaBase { return *ColumnBlobBytes + *IndexBlobBytes; } - const TBlobRange RestoreBlobRange(const TBlobRangeLink16& linkRange) const { - return linkRange.RestoreRange(GetBlobId(linkRange.GetBlobIdxVerified())); - } - - TBlobRangeLink16::TLinkId RegisterBlobId(const TUnifiedBlobId& blobId) { - AFL_VERIFY(blobId.IsValid()); - TBlobRangeLink16::TLinkId idx = 0; - for (auto&& i : BlobIds) { - if (i == blobId) { - return idx; - } - ++idx; - } - BlobIds.emplace_back(blobId); - return idx; - } - void SetCompactionLevel(const ui64 level) { CompactionLevel = level; } diff --git a/ydb/core/tx/columnshard/engines/portions/constructor_portion.h b/ydb/core/tx/columnshard/engines/portions/constructor_portion.h index ed028161f667..0212be824231 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor_portion.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor_portion.h @@ -35,7 +35,7 @@ class TPortionInfoConstructor { , RemoveSnapshot(portion.GetRemoveSnapshotOptional()) , SchemaVersion(portion.GetSchemaVersionVerified()) , ShardingVersion(portion.GetShardingVersionOptional()) { - MetaConstructor = TPortionMetaConstructor(std::move(portion.Meta), true); + MetaConstructor = TPortionMetaConstructor(std::move(portion.Meta)); } virtual std::shared_ptr BuildPortionImpl(TPortionMeta&& meta) = 0; @@ -50,23 +50,17 @@ class TPortionInfoConstructor { virtual EPortionType GetType() const = 0; - TPortionInfoConstructor(const TPortionInfo& portion, const bool withMetadata, const bool withMetadataBlobs) + TPortionInfoConstructor(const TPortionInfo& portion, const bool withMetadata) : PathId(portion.GetPathId()) , PortionId(portion.GetPortionId()) , RemoveSnapshot(portion.GetRemoveSnapshotOptional()) , SchemaVersion(portion.GetSchemaVersionVerified()) , ShardingVersion(portion.GetShardingVersionOptional()) { if (withMetadata) { - MetaConstructor = TPortionMetaConstructor(portion.Meta, withMetadataBlobs); - } else { - AFL_VERIFY(!withMetadataBlobs); + MetaConstructor = TPortionMetaConstructor(portion.Meta); } } - bool HaveBlobsData() { - return MetaConstructor.GetBlobIdsCount(); - } - void SetPortionId(const ui64 value) { AFL_VERIFY(value); PortionId = value; @@ -154,8 +148,8 @@ class TCompactedPortionInfoConstructor: public TPortionInfoConstructor { public: using TBase::TBase; - TCompactedPortionInfoConstructor(const TCompactedPortionInfo& portion, const bool withMetadata, const bool withMetadataBlobs) - : TBase(portion, withMetadata, withMetadataBlobs) { + TCompactedPortionInfoConstructor(const TCompactedPortionInfo& portion, const bool withMetadata) + : TBase(portion, withMetadata) { } virtual EPortionType GetType() const override { @@ -180,8 +174,8 @@ class TWrittenPortionInfoConstructor: public TPortionInfoConstructor { return EPortionType::Written; } - TWrittenPortionInfoConstructor(const TWrittenPortionInfo& portion, const bool withMetadata, const bool withMetadataBlobs) - : TBase(portion, withMetadata, withMetadataBlobs) + TWrittenPortionInfoConstructor(const TWrittenPortionInfo& portion, const bool withMetadata) + : TBase(portion, withMetadata) , CommitSnapshot(portion.GetCommitSnapshotOptional()) , InsertWriteId(portion.GetInsertWriteId()) { } diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp index 53323737b126..d10a048800ac 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp @@ -95,8 +95,8 @@ void TPortionDataAccessor::FillBlobRangesByStorage( continue; } const TString& storageId = PortionInfo->GetColumnStorageId(i.GetColumnId(), indexInfo); - AFL_VERIFY(result[i.GetEntityId()][storageId].emplace(PortionInfo->RestoreBlobRange(i.GetBlobRange())).second)( - "blob_id", PortionInfo->RestoreBlobRange(i.GetBlobRange()).ToString()); + AFL_VERIFY(result[i.GetEntityId()][storageId].emplace(RestoreBlobRange(i.GetBlobRange())).second)( + "blob_id", RestoreBlobRange(i.GetBlobRange()).ToString()); } for (auto&& i : GetIndexesVerified()) { if (!entityIds.contains(i.GetEntityId())) { @@ -104,8 +104,7 @@ void TPortionDataAccessor::FillBlobRangesByStorage( } const TString& storageId = PortionInfo->GetIndexStorageId(i.GetIndexId(), indexInfo); auto bRange = i.GetBlobRangeVerified(); - AFL_VERIFY(result[i.GetEntityId()][storageId].emplace(PortionInfo->RestoreBlobRange(bRange)).second)( - "blob_id", PortionInfo->RestoreBlobRange(bRange).ToString()); + AFL_VERIFY(result[i.GetEntityId()][storageId].emplace(RestoreBlobRange(bRange)).second)("blob_id", RestoreBlobRange(bRange).ToString()); } } @@ -114,14 +113,15 @@ void TPortionDataAccessor::FillBlobRangesByStorage(THashMapGetIndexInfo()); } -void TPortionDataAccessor::FillBlobRangesByStorage(THashMap>& result, const TIndexInfo& indexInfo, const std::set* entityIds) const { +void TPortionDataAccessor::FillBlobRangesByStorage( + THashMap>& result, const TIndexInfo& indexInfo, const std::set* entityIds) const { for (auto&& i : GetRecordsVerified()) { if (entityIds && !entityIds->contains(i.GetColumnId())) { continue; } const TString& storageId = PortionInfo->GetColumnStorageId(i.GetColumnId(), indexInfo); - AFL_VERIFY(result[storageId].emplace(PortionInfo->RestoreBlobRange(i.GetBlobRange())).second)( - "blob_id", PortionInfo->RestoreBlobRange(i.GetBlobRange()).ToString()); + AFL_VERIFY(result[storageId].emplace(RestoreBlobRange(i.GetBlobRange())).second)( + "blob_id", RestoreBlobRange(i.GetBlobRange()).ToString()); } for (auto&& i : GetIndexesVerified()) { if (entityIds && !entityIds->contains(i.GetIndexId())) { @@ -129,8 +129,7 @@ void TPortionDataAccessor::FillBlobRangesByStorage(THashMapGetIndexStorageId(i.GetIndexId(), indexInfo); if (auto bRange = i.GetBlobRangeOptional()) { - AFL_VERIFY(result[storageId].emplace(PortionInfo->RestoreBlobRange(*bRange)).second)( - "blob_id", PortionInfo->RestoreBlobRange(*bRange).ToString()); + AFL_VERIFY(result[storageId].emplace(RestoreBlobRange(*bRange)).second)("blob_id", RestoreBlobRange(*bRange).ToString()); } } } @@ -141,7 +140,7 @@ void TPortionDataAccessor::FillBlobIdsByStorage(THashMap* currentHashResult = nullptr; std::optional lastEntityId; TString lastStorageId; - ui32 lastBlobIdx = PortionInfo->GetBlobIdsCount(); + ui32 lastBlobIdx = GetBlobIdsCount(); for (auto&& i : GetRecordsVerified()) { if (!lastEntityId || *lastEntityId != i.GetEntityId()) { const TString& storageId = PortionInfo->GetColumnStorageId(i.GetEntityId(), indexInfo); @@ -150,11 +149,11 @@ void TPortionDataAccessor::FillBlobIdsByStorage(THashMapGetBlobIdsCount(); + lastBlobIdx = GetBlobIdsCount(); } } if (lastBlobIdx != i.GetBlobRange().GetBlobIdxVerified() && currentHashLocal->emplace(i.GetBlobRange().GetBlobIdxVerified()).second) { - auto blobId = PortionInfo->GetBlobId(i.GetBlobRange().GetBlobIdxVerified()); + auto blobId = GetBlobId(i.GetBlobRange().GetBlobIdxVerified()); AFL_VERIFY(currentHashResult); AFL_VERIFY(currentHashResult->emplace(blobId).second)("blob_id", blobId.ToStringNew()); lastBlobIdx = i.GetBlobRange().GetBlobIdxVerified(); @@ -168,12 +167,12 @@ void TPortionDataAccessor::FillBlobIdsByStorage(THashMapGetBlobIdsCount(); + lastBlobIdx = GetBlobIdsCount(); } } if (auto bRange = i.GetBlobRangeOptional()) { if (lastBlobIdx != bRange->GetBlobIdxVerified() && currentHashLocal->emplace(bRange->GetBlobIdxVerified()).second) { - auto blobId = PortionInfo->GetBlobId(bRange->GetBlobIdxVerified()); + auto blobId = GetBlobId(bRange->GetBlobIdxVerified()); AFL_VERIFY(currentHashResult); AFL_VERIFY(currentHashResult->emplace(blobId).second)("blob_id", blobId.ToStringNew()); lastBlobIdx = bRange->GetBlobIdxVerified(); @@ -193,7 +192,7 @@ THashMap>> T for (auto&& c : GetRecordsVerified()) { const TString& storageId = PortionInfo->GetColumnStorageId(c.GetColumnId(), indexInfo); auto chunk = std::make_shared( - blobs.Extract(storageId, PortionInfo->RestoreBlobRange(c.GetBlobRange())), c, indexInfo.GetColumnFeaturesVerified(c.GetColumnId())); + blobs.Extract(storageId, RestoreBlobRange(c.GetBlobRange())), c, indexInfo.GetColumnFeaturesVerified(c.GetColumnId())); chunk->SetChunkIdx(c.GetChunkIdx()); AFL_VERIFY(result[storageId].emplace(c.GetAddress(), chunk).second); } @@ -201,7 +200,7 @@ THashMap>> T const TString& storageId = indexInfo.GetIndexStorageId(c.GetIndexId()); const TString blobData = [&]() -> TString { if (auto bRange = c.GetBlobRangeOptional()) { - return blobs.Extract(storageId, PortionInfo->RestoreBlobRange(*bRange)); + return blobs.Extract(storageId, RestoreBlobRange(*bRange)); } else if (auto data = c.GetBlobDataOptional()) { return *data; } else { @@ -226,7 +225,7 @@ THashMap TPortionDataAccessor::DecodeBlobAddresses( TString columnStorageId; ui32 columnId = 0; for (auto&& record : GetRecordsVerified()) { - if (PortionInfo->RestoreBlobRange(record.GetBlobRange()) == b.first) { + if (RestoreBlobRange(record.GetBlobRange()) == b.first) { if (columnId != record.GetColumnId()) { columnStorageId = PortionInfo->GetColumnStorageId(record.GetColumnId(), indexInfo); } @@ -245,7 +244,7 @@ THashMap TPortionDataAccessor::DecodeBlobAddresses( if (!record.HasBlobRange()) { continue; } - if (PortionInfo->RestoreBlobRange(record.GetBlobRangeVerified()) == b.first) { + if (RestoreBlobRange(record.GetBlobRangeVerified()) == b.first) { if (columnId != record.GetIndexId()) { columnStorageId = indexInfo.GetIndexStorageId(record.GetIndexId()); } @@ -302,7 +301,7 @@ TString TPortionDataAccessor::DebugString() const { if (IS_TRACE_LOG_ENABLED(NKikimrServices::TX_COLUMNSHARD)) { std::vector blobRanges; for (auto&& i : GetRecordsVerified()) { - blobRanges.emplace_back(PortionInfo->RestoreBlobRange(i.BlobRange)); + blobRanges.emplace_back(RestoreBlobRange(i.BlobRange)); } sb << "blobs:" << JoinSeq(",", blobRanges) << ";ranges_count:" << blobRanges.size() << ";"; } @@ -621,19 +620,24 @@ ui64 TPortionDataAccessor::GetMinMemoryForReadColumns(const std::optionalFullValidation(); std::set blobIdxs; for (auto&& i : GetRecordsVerified()) { - TBlobRange::Validate(PortionInfo->GetMeta().GetBlobIds(), i.GetBlobRange()).Validate(); + TBlobRange::Validate(BlobIds, i.GetBlobRange()).Validate(); blobIdxs.emplace(i.GetBlobRange().GetBlobIdxVerified()); } AFL_VERIFY(GetRecordsVerified().size()); for (auto&& i : GetIndexesVerified()) { if (auto bRange = i.GetBlobRangeOptional()) { - TBlobRange::Validate(PortionInfo->GetMeta().GetBlobIds(), *bRange).Validate(); + TBlobRange::Validate(BlobIds, *bRange).Validate(); blobIdxs.emplace(bRange->GetBlobIdxVerified()); } } AFL_VERIFY(blobIdxs.size())("portion_info", PortionInfo->DebugString()); - AFL_VERIFY(PortionInfo->GetBlobIdsCount() == blobIdxs.size()); - AFL_VERIFY(PortionInfo->GetBlobIdsCount() == *blobIdxs.rbegin() + 1); + AFL_VERIFY(BlobIds.size() == blobIdxs.size()); + AFL_VERIFY(BlobIds.size() == *blobIdxs.rbegin() + 1); } void TPortionDataAccessor::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const { - PortionInfo->SerializeToProto(proto); + PortionInfo->SerializeToProto(GetBlobIds(), proto); AFL_VERIFY(GetRecordsVerified().size()); for (auto&& r : GetRecordsVerified()) { *proto.AddRecords() = r.SerializeToProto(); diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index 0d96af82e5ac..5e827867662d 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -203,10 +203,11 @@ class TColumnAssemblingInfo { } }; -} +} // namespace NAssembling -class TPortionDataAccessor { +class TPortionDataAccessor: public TPortionMetaBase { private: + using TBase = TPortionMetaBase; TPortionInfo::TConstPtr PortionInfo; std::optional> Records; std::optional> Indexes; @@ -238,9 +239,8 @@ class TPortionDataAccessor { using TPreparedBatchData = NAssembling::TPreparedBatchData; ui64 GetMetadataSize() const { - return (Records ? (Records->size() * sizeof(TColumnRecord)) : 0) + - (Indexes ? (Indexes->size() * sizeof(TIndexChunk)) : 0) + - sizeof(TPortionInfo::TConstPtr); + return (Records ? (Records->size() * sizeof(TColumnRecord)) : 0) + (Indexes ? (Indexes->size() * sizeof(TIndexChunk)) : 0) + + sizeof(TPortionInfo::TConstPtr) + TBase::GetMetadataMemorySize(); } class TExtractContext { @@ -295,7 +295,8 @@ class TPortionDataAccessor { extractedIndexes = *Indexes; } - return TPortionDataAccessor(PortionInfo, std::move(extractedRecords), std::move(extractedIndexes), false); + std::vector blobIds = BlobIds; + return TPortionDataAccessor(PortionInfo, std::move(blobIds), std::move(extractedRecords), std::move(extractedIndexes), false); } const std::vector& TestGetRecords() const { @@ -312,7 +313,7 @@ class TPortionDataAccessor { } TPortionDataAccessor SwitchPortionInfo(std::shared_ptr&& newPortion) const { - return TPortionDataAccessor(newPortion, GetRecordsVerified(), GetIndexesVerified(), true); + return TPortionDataAccessor(newPortion, BlobIds, GetRecordsVerified(), GetIndexesVerified(), true); } template @@ -343,19 +344,22 @@ class TPortionDataAccessor { } } - explicit TPortionDataAccessor(const TPortionInfo::TConstPtr& portionInfo, std::vector&& records, - std::vector&& indexes, const bool validate) - : PortionInfo(portionInfo) + explicit TPortionDataAccessor(const TPortionInfo::TConstPtr& portionInfo, std::vector&& blobIds, + std::vector&& records, std::vector&& indexes, const bool validate) + : TBase(std::move(blobIds)) + , PortionInfo(portionInfo) , Records(std::move(records)) , Indexes(std::move(indexes)) { + AFL_VERIFY(BlobIds.size()); if (validate) { FullValidation(); } } - explicit TPortionDataAccessor(const TPortionInfo::TConstPtr& portionInfo, const std::vector& records, - const std::vector& indexes, const bool validate) - : PortionInfo(portionInfo) + explicit TPortionDataAccessor(const TPortionInfo::TConstPtr& portionInfo, const std::vector& blobIds, + const std::vector& records, const std::vector& indexes, const bool validate) + : TBase(blobIds) + , PortionInfo(portionInfo) , Records(records) , Indexes(indexes) { if (validate) { @@ -428,7 +432,8 @@ class TPortionDataAccessor { void FillBlobRangesByStorage( THashMap>& result, const TIndexInfo& indexInfo, const std::set* entityIds = nullptr) const; void FillBlobRangesByStorage(THashMap>& result, const TVersionedIndex& index) const; - void FillBlobRangesByStorage(THashMap>>& result, const TIndexInfo& indexInfo, const THashSet& entityIds) const; + void FillBlobRangesByStorage( + THashMap>>& result, const TIndexInfo& indexInfo, const THashSet& entityIds) const; void FillBlobRangesByStorage( THashMap>>& result, const TVersionedIndex& index, const THashSet& entityIds) const; void FillBlobIdsByStorage(THashMap>& result, const TIndexInfo& indexInfo) const; @@ -473,7 +478,8 @@ class TPortionDataAccessor { THashMap& blobsData, const std::optional& defaultSnapshot = std::nullopt, const bool restoreAbsent = true) const; TPreparedBatchData PrepareForAssemble(const ISnapshotSchema& dataSchema, const ISnapshotSchema& resultSchema, - THashMap& blobsData, const std::optional& defaultSnapshot = std::nullopt, const bool restoreAbsent = true) const; + THashMap& blobsData, const std::optional& defaultSnapshot = std::nullopt, + const bool restoreAbsent = true) const; class TPage { private: diff --git a/ydb/core/tx/columnshard/engines/portions/meta.cpp b/ydb/core/tx/columnshard/engines/portions/meta.cpp index b0811125f974..8f0bced783a5 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.cpp +++ b/ydb/core/tx/columnshard/engines/portions/meta.cpp @@ -10,7 +10,8 @@ namespace NKikimr::NOlap { -NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto(const NPortion::EProduced produced) const { +NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto(const std::vector& blobIds, const NPortion::EProduced produced) const { + AFL_VERIFY(blobIds.size()); FullValidation(); NKikimrTxColumnShard::TIndexPortionMeta portionMeta; portionMeta.SetTierName(TierName); @@ -51,7 +52,7 @@ NKikimrTxColumnShard::TIndexPortionMeta TPortionMeta::SerializeToProto(const NPo RecordSnapshotMin.SerializeToProto(*portionMeta.MutableRecordSnapshotMin()); RecordSnapshotMax.SerializeToProto(*portionMeta.MutableRecordSnapshotMax()); - for (auto&& i : GetBlobIds()) { + for (auto&& i : blobIds) { *portionMeta.AddBlobIds() = i.GetLogoBlobId().AsBinaryString(); } return portionMeta; diff --git a/ydb/core/tx/columnshard/engines/portions/meta.h b/ydb/core/tx/columnshard/engines/portions/meta.h index d56619e99566..7ad191e972b9 100644 --- a/ydb/core/tx/columnshard/engines/portions/meta.h +++ b/ydb/core/tx/columnshard/engines/portions/meta.h @@ -1,13 +1,13 @@ #pragma once #include #include +#include #include #include #include #include #include -#include #include @@ -18,19 +18,6 @@ struct TIndexInfo; class TPortionMetaBase { protected: std::vector BlobIds; -public: - const std::vector& GetBlobIds() const { - return BlobIds; - } - - const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const { - AFL_VERIFY(linkId < GetBlobIds().size()); - return BlobIds[linkId]; - } - - ui32 GetBlobIdsCount() const { - return BlobIds.size(); - } void FullValidation() const { for (auto&& i : BlobIds) { @@ -51,24 +38,64 @@ class TPortionMetaBase { return std::nullopt; } - ui64 GetMetadataMemorySize() const { - return GetBlobIds().capacity() * sizeof(TUnifiedBlobId); + TBlobRangeLink16::TLinkId GetBlobIdxVerified(const TUnifiedBlobId& blobId) const { + auto result = GetBlobIdxOptional(blobId); + AFL_VERIFY(result); + return *result; } - ui64 GetMetadataDataSize() const { - return GetBlobIds().size() * sizeof(TUnifiedBlobId); +public: + TPortionMetaBase() = default; + + const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const { + AFL_VERIFY(linkId < GetBlobIds().size()); + return BlobIds[linkId]; + } + const std::vector& GetBlobIds() const { + return BlobIds; } - TBlobRangeLink16::TLinkId GetBlobIdxVerified(const TUnifiedBlobId& blobId) const { + ui32 GetBlobIdsCount() const { + return BlobIds.size(); + } + + TPortionMetaBase(std::vector&& blobIds) + : BlobIds(std::move(blobIds)) { + } + + TPortionMetaBase(const std::vector& blobIds) + : BlobIds(blobIds) { + } + + std::vector ExtractBlobIds() { + return std::move(BlobIds); + } + + TBlobRangeLink16::TLinkId GetBlobIdxVerifiedPrivate(const TUnifiedBlobId& blobId) const { auto result = GetBlobIdxOptional(blobId); AFL_VERIFY(result); return *result; } + + const std::vector& GetBlobIdsPrivate() const { + return BlobIds; + } + + const TBlobRange RestoreBlobRange(const TBlobRangeLink16& linkRange) const { + return linkRange.RestoreRange(GetBlobId(linkRange.GetBlobIdxVerified())); + } + + ui64 GetMetadataMemorySize() const { + return GetBlobIds().capacity() * sizeof(TUnifiedBlobId); + } + + ui64 GetMetadataDataSize() const { + return GetBlobIds().size() * sizeof(TUnifiedBlobId); + } }; -class TPortionMeta: public TPortionMetaBase { +class TPortionMeta { private: - using TBase = TPortionMetaBase; std::shared_ptr PKSchema; NArrow::TSimpleRowContent FirstPKRow; NArrow::TSimpleRowContent LastPKRow; @@ -82,6 +109,7 @@ class TPortionMeta: public TPortionMetaBase { YDB_READONLY(ui32, IndexBlobBytes, 0); friend class TPortionMetaConstructor; + friend class TPortionInfo; friend class TCompactedPortionInfo; TPortionMeta(NArrow::TFirstLastSpecialKeys& pk, const TSnapshot& min, const TSnapshot& max) : PKSchema(pk.GetSchema()) @@ -95,9 +123,7 @@ class TPortionMeta: public TPortionMetaBase { TSnapshot RecordSnapshotMax; public: - void FullValidation() const { - TBase::FullValidation(); AFL_VERIFY(RecordsCount); AFL_VERIFY(ColumnRawBytes); AFL_VERIFY(ColumnBlobBytes); @@ -122,14 +148,15 @@ class TPortionMeta: public TPortionMetaBase { } ui64 GetMemorySize() const { - return sizeof(TPortionMeta) + FirstPKRow.GetMemorySize() + LastPKRow.GetMemorySize() + TBase::GetMetadataMemorySize(); + return sizeof(TPortionMeta) + FirstPKRow.GetMemorySize() + LastPKRow.GetMemorySize(); } ui64 GetDataSize() const { - return sizeof(TPortionMeta) + FirstPKRow.GetDataSize() + LastPKRow.GetDataSize() + TBase::GetMetadataDataSize(); + return sizeof(TPortionMeta) + FirstPKRow.GetDataSize() + LastPKRow.GetDataSize(); } - NKikimrTxColumnShard::TIndexPortionMeta SerializeToProto(const NPortion::EProduced produced) const; + NKikimrTxColumnShard::TIndexPortionMeta SerializeToProto( + const std::vector& blobIds, const NPortion::EProduced produced) const; TString DebugString() const; }; diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index dff837746e95..b827d6e19472 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -52,7 +52,7 @@ ui64 TPortionInfo::GetApproxChunksCount(const ui32 schemaColumnsCount) const { return schemaColumnsCount * (GetRecordsCount() / 10000 + 1); } -void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const { +void TPortionInfo::SerializeToProto(const std::vector& blobIds, NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const { PathId.ToProto(proto); proto.SetPortionId(PortionId); proto.SetSchemaVersion(GetSchemaVersionVerified()); @@ -60,7 +60,7 @@ void TPortionInfo::SerializeToProto(NKikimrColumnShardDataSharingProto::TPortion *proto.MutableRemoveSnapshot() = RemoveSnapshot.SerializeToProto(); } - *proto.MutableMeta() = Meta.SerializeToProto(GetProduced()); + *proto.MutableMeta() = Meta.SerializeToProto(blobIds, GetProduced()); } TConclusionStatus TPortionInfo::DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 33543d697410..69431ef1d2b4 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -80,7 +80,6 @@ class TPortionInfo { }; private: - friend class TPortionDataAccessor; friend class TPortionInfoConstructor; friend class TCompactedPortionInfo; friend class TWrittenPortionInfo; @@ -97,6 +96,14 @@ class TPortionInfo { TPortionMeta Meta; TRuntimeFeatures RuntimeFeatures = 0; + virtual void DoSaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const = 0; + + virtual bool DoIsVisible(const TSnapshot& snapshot, const bool checkCommitSnapshot) const = 0; + virtual TString DoDebugString(const bool /*withDetails*/) const { + return ""; + } + +public: void FullValidation() const { AFL_VERIFY(PathId); AFL_VERIFY(PortionId); @@ -106,14 +113,6 @@ class TPortionInfo { TConclusionStatus DeserializeFromProto(const NKikimrColumnShardDataSharingProto::TPortionInfo& proto); - virtual void DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const = 0; - - virtual bool DoIsVisible(const TSnapshot& snapshot, const bool checkCommitSnapshot) const = 0; - virtual TString DoDebugString(const bool /*withDetails*/) const { - return ""; - } - -public: virtual EPortionType GetPortionType() const = 0; virtual bool IsCommitted() const = 0; NPortion::TPortionInfoForCompaction GetCompactionInfo() const { @@ -149,16 +148,12 @@ class TPortionInfo { return (GetRecordsCount() / 10000 + 1) * sizeof(TColumnRecord) * columnsCount; } - void SaveMetaToDatabase(NIceDb::TNiceDb& db) const { + void SaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const { FullValidation(); - DoSaveMetaToDatabase(db); + DoSaveMetaToDatabase(blobIds, db); } - virtual std::unique_ptr BuildConstructor(const bool withMetadata, const bool withMetadataBlobs) const = 0; - - const std::vector& GetBlobIds() const { - return Meta.GetBlobIds(); - } + virtual std::unique_ptr BuildConstructor(const bool withMetadata) const = 0; ui32 GetCompactionLevel() const { return GetMeta().GetCompactionLevel(); @@ -227,18 +222,6 @@ class TPortionInfo { return (RuntimeFeatures & (TRuntimeFeatures)feature); } - const TBlobRange RestoreBlobRange(const TBlobRangeLink16& linkRange) const { - return linkRange.RestoreRange(GetBlobId(linkRange.GetBlobIdxVerified())); - } - - const TUnifiedBlobId& GetBlobId(const TBlobRangeLink16::TLinkId linkId) const { - return Meta.GetBlobId(linkId); - } - - ui32 GetBlobIdsCount() const { - return Meta.GetBlobIdsCount(); - } - ui64 GetTxVolume() const { return 1024; } @@ -246,7 +229,7 @@ class TPortionInfo { ui64 GetApproxChunksCount(const ui32 schemaColumnsCount) const; ui64 GetMetadataMemorySize() const; - void SerializeToProto(NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const; + void SerializeToProto(const std::vector& blobIds, NKikimrColumnShardDataSharingProto::TPortionInfo& proto) const; TInternalPathId GetPathId() const { return PathId; diff --git a/ydb/core/tx/columnshard/engines/portions/written.cpp b/ydb/core/tx/columnshard/engines/portions/written.cpp index 5db336d99be1..09ea1bff7ac5 100644 --- a/ydb/core/tx/columnshard/engines/portions/written.cpp +++ b/ydb/core/tx/columnshard/engines/portions/written.cpp @@ -3,11 +3,12 @@ #include "written.h" #include +#include namespace NKikimr::NOlap { -void TWrittenPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const { - auto metaProto = GetMeta().SerializeToProto(NPortion::EProduced::INSERTED); +void TWrittenPortionInfo::DoSaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const { + auto metaProto = GetMeta().SerializeToProto(blobIds, NPortion::EProduced::INSERTED); using IndexPortions = NColumnShard::Schema::IndexPortions; const auto removeSnapshot = GetRemoveSnapshotOptional(); AFL_VERIFY(InsertWriteId); @@ -20,13 +21,12 @@ void TWrittenPortionInfo::DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const { NIceDb::TUpdate((ui64)*InsertWriteId), NIceDb::TUpdate(removeSnapshot ? removeSnapshot->GetPlanStep() : 0), NIceDb::TUpdate(removeSnapshot ? removeSnapshot->GetTxId() : 0), - NIceDb::TUpdate(1), - NIceDb::TUpdate(1), + NIceDb::TUpdate(1), NIceDb::TUpdate(1), NIceDb::TUpdate(metaProto.SerializeAsString())); } -std::unique_ptr TWrittenPortionInfo::BuildConstructor(const bool withMetadata, const bool withMetadataBlobs) const { - return std::make_unique(*this, withMetadata, withMetadataBlobs); +std::unique_ptr TWrittenPortionInfo::BuildConstructor(const bool withMetadata) const { + return std::make_unique(*this, withMetadata); } void TWrittenPortionInfo::FillDefaultColumn(NAssembling::TColumnAssemblingInfo& column, const std::optional& defaultSnapshot) const { @@ -70,4 +70,9 @@ bool TWrittenPortionInfo::DoIsVisible(const TSnapshot& snapshot, const bool chec } } +void TWrittenPortionInfo::CommitToDatabase(IDbWrapper& wrapper) { + AFL_VERIFY(CommitSnapshot); + wrapper.CommitPortion(*this, *CommitSnapshot); +} + } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/portions/written.h b/ydb/core/tx/columnshard/engines/portions/written.h index ca8be72a0295..9bc369518578 100644 --- a/ydb/core/tx/columnshard/engines/portions/written.h +++ b/ydb/core/tx/columnshard/engines/portions/written.h @@ -4,6 +4,7 @@ namespace NKikimr::NOlap { class TWrittenPortionInfoConstructor; +class IDbWrapper; class TWrittenPortionInfo: public TPortionInfo { private: @@ -12,7 +13,7 @@ class TWrittenPortionInfo: public TPortionInfo { std::optional InsertWriteId; friend class TWrittenPortionInfoConstructor; - virtual void DoSaveMetaToDatabase(NIceDb::TNiceDb& db) const override; + virtual void DoSaveMetaToDatabase(const std::vector& blobIds, NIceDb::TNiceDb& db) const override; virtual EPortionType GetPortionType() const override { return EPortionType::Written; @@ -42,6 +43,8 @@ class TWrittenPortionInfo: public TPortionInfo { public: virtual void FillDefaultColumn(NAssembling::TColumnAssemblingInfo& column, const std::optional& defaultSnapshot) const override; + void CommitToDatabase(IDbWrapper& wrapper); + virtual NSplitter::TEntityGroups GetEntityGroupsByStorageId( const TString& /*specialTier*/, const IStoragesManager& storages, const TIndexInfo& /*indexInfo*/) const override { NSplitter::TEntityGroups groups(storages.GetDefaultOperator()->GetBlobSplitSettings(), IStoragesManager::DefaultStorageId); @@ -60,7 +63,7 @@ class TWrittenPortionInfo: public TPortionInfo { return { NBlobOperations::TGlobal::DefaultStorageId }; } - virtual std::unique_ptr BuildConstructor(const bool withMetadata, const bool withMetadataBlobs) const override; + virtual std::unique_ptr BuildConstructor(const bool withMetadata) const override; TWrittenPortionInfo(TPortionMeta&& meta) : TBase(std::move(meta)) { diff --git a/ydb/core/tx/columnshard/engines/protos/portion_info.proto b/ydb/core/tx/columnshard/engines/protos/portion_info.proto index 8b399b8b19c3..aec83a713d01 100644 --- a/ydb/core/tx/columnshard/engines/protos/portion_info.proto +++ b/ydb/core/tx/columnshard/engines/protos/portion_info.proto @@ -53,3 +53,7 @@ message TColumnChunkInfo { message TIndexPortionAccessor { repeated TColumnChunkInfo Chunks = 1; } + +message TIndexPortionBlobsInfo { + repeated string BlobIds = 1; +} diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 4c259b235b7d..89c83c3ee306 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -94,7 +94,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob if (!itFilter.IsBatchForSkip(c->GetMeta().GetRecordsCount())) { auto reading = blobsAction.GetReading(Portion->GetColumnStorageId(c->GetColumnId(), Schema->GetIndexInfo())); reading->SetIsBackgroundProcess(false); - reading->AddRange(Portion->RestoreBlobRange(c->BlobRange)); + reading->AddRange(GetStageData().GetPortionAccessor().RestoreBlobRange(c->BlobRange)); ++fetchedChunks; } else { defaultBlocks.emplace(c->GetAddress(), TPortionDataAccessor::TAssembleBlobInfo(c->GetMeta().GetRecordsCount(), diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index a20d9395a5bf..6262f5e50455 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -112,7 +112,7 @@ void TPortionDataSource::NeedFetchColumns(const std::set& columnIds, TBlob if (!itFilter.IsBatchForSkip(c->GetMeta().GetRecordsCount())) { auto reading = blobsAction.GetReading(Portion->GetColumnStorageId(c->GetColumnId(), Schema->GetIndexInfo())); reading->SetIsBackgroundProcess(false); - reading->AddRange(Portion->RestoreBlobRange(c->BlobRange)); + reading->AddRange(GetStageData().GetPortionAccessor().RestoreBlobRange(c->BlobRange)); ++fetchedChunks; } else { defaultBlocks.emplace(c->GetAddress(), TPortionDataAccessor::TAssembleBlobInfo(c->GetMeta().GetRecordsCount(), diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h index d2a0d4564339..a681e8d69c40 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.h @@ -385,7 +385,7 @@ class TPortionDataSource: public IDataSource { } virtual TBlobRange RestoreBlobRange(const TBlobRangeLink16& rangeLink) const override { - return Portion->RestoreBlobRange(rangeLink); + return GetStageData().GetPortionAccessor().RestoreBlobRange(rangeLink); } virtual const std::shared_ptr& GetSourceSchema() const override { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index b6499e30ad34..d602958b80ee 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -71,7 +71,7 @@ void TStatsIterator::AppendStats(const std::vectorGetBlobRange().GetBlobIdxVerified(), - portion.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy()) + portionPtr.GetBlobId(r->GetBlobRange().GetBlobIdxVerified()).ToStringLegacy()) .first; } NArrow::Append( @@ -104,7 +104,7 @@ void TStatsIterator::AppendStats(const std::vector(*builders[7], ReadMetadata->GetEntityName(r->GetIndexId()).value_or("undefined")); NArrow::Append(*builders[8], r->GetIndexId()); if (auto bRange = r->GetBlobRangeOptional()) { - std::string blobIdString = portion.GetBlobId(bRange->GetBlobIdxVerified()).ToStringLegacy(); + std::string blobIdString = portionPtr.GetBlobId(bRange->GetBlobIdxVerified()).ToStringLegacy(); NArrow::Append(*builders[9], blobIdString); NArrow::Append(*builders[10], bRange->Offset); NArrow::Append(*builders[11], bRange->Size); diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/fetcher.cpp b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/fetcher.cpp index a7751f155bd6..394e24e7df15 100644 --- a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/fetcher.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/fetcher.cpp @@ -13,7 +13,7 @@ void TIndexFetcherLogic::DoStart(TReadActionsCollection& nextRead, NReader::NCom StorageId, IndexAddressesVector, originalData, IndexMeta->BuildHeader(originalData).DetachResult(), i->GetRecordsCount())); } else { TChunkOriginalData originalData( - context.GetSource()->GetStageData().GetPortionAccessor().GetPortionInfo().RestoreBlobRange(i->GetBlobRangeVerified())); + context.GetSource()->GetStageData().GetPortionAccessor().RestoreBlobRange(i->GetBlobRangeVerified())); Fetching.emplace_back(TIndexChunkFetching( StorageId, IndexAddressesVector, originalData, IndexMeta->BuildHeader(originalData).DetachResult(), i->GetRecordsCount())); } diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp index f18e8b876fb1..5f09bbcbbb25 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.cpp @@ -247,9 +247,7 @@ bool TGranuleMeta::TestingLoad(IDbWrapper& db, const TVersionedIndex& versionedI { if (!db.LoadColumns(PathId, [&](TColumnChunkLoadContextV2&& loadContext) { auto* constructor = constructors.GetConstructorVerified(loadContext.GetPortionId()); - for (auto&& i : loadContext.BuildRecordsV1()) { - constructor->LoadRecord(std::move(i)); - } + constructor->AddBuildInfo(loadContext.CreateBuildInfo()); })) { return false; } @@ -300,7 +298,7 @@ void TGranuleMeta::CommitPortionOnExecute( AFL_VERIFY(it != InsertedPortions.end()); it->second->SetCommitSnapshot(snapshot); TDbWrapper wrapper(txc.DB, nullptr); - wrapper.WritePortion(*it->second); + it->second->CommitToDatabase(wrapper); } void TGranuleMeta::CommitPortionOnComplete(const TInsertWriteId insertWriteId, IColumnEngine& engine) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/granule.h b/ydb/core/tx/columnshard/engines/storage/granule/granule.h index 11b0c79cd859..af406e34fec0 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/granule.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/granule.h @@ -199,7 +199,7 @@ class TGranuleMeta: TNonCopyable { auto accessorCopy = portion.SwitchPortionInfo(std::move(copy)); accessorCopy.SaveToDatabase(wrapper, firstPKColumnId, false); } else { - wrapper.WritePortion(*copy); + wrapper.WritePortion(portion.GetBlobIds(), *copy); } } @@ -228,7 +228,7 @@ class TGranuleMeta: TNonCopyable { it->second->SetCommitSnapshot(ssRemove); it->second->SetRemoveSnapshot(ssRemove); TDbWrapper wrapper(txc.DB, nullptr); - wrapper.WritePortion(*it->second); + it->second->CommitToDatabase(wrapper); } void AbortPortionOnComplete(const TInsertWriteId insertWriteId, IColumnEngine& engine) { diff --git a/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp b/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp index 48c7963c26c6..f00764254288 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp +++ b/ydb/core/tx/columnshard/engines/storage/granule/stages.cpp @@ -9,11 +9,12 @@ namespace NKikimr::NOlap::NLoading { bool TGranuleOnlyPortionsReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { TDbWrapper db(txc.DB, &*DsGroupSelector); std::vector portions; - if (!db.LoadPortions(Self->GetPathId(), [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { - const TIndexInfo& indexInfo = portion->GetSchema(*VersionedIndex)->GetIndexInfo(); - AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, *DsGroupSelector)); - portions.emplace_back(portion->Build()); - })) { + if (!db.LoadPortions(Self->GetPathId(), + [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = portion->GetSchema(*VersionedIndex)->GetIndexInfo(); + AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, *DsGroupSelector)); + portions.emplace_back(portion->Build()); + })) { return false; } for (auto&& i : portions) { @@ -29,7 +30,6 @@ bool TGranuleOnlyPortionsReader::DoPrecharge(NTabletFlatExecutor::TTransactionCo bool TGranuleColumnsReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { TDbWrapper db(txc.DB, &*DsGroupSelector); - Context->ClearRecords(); return db.LoadColumns(Self->GetPathId(), [&](TColumnChunkLoadContextV2&& loadContext) { Context->Add(std::move(loadContext)); }); @@ -42,10 +42,10 @@ bool TGranuleColumnsReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext bool TGranuleIndexesReader::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { TDbWrapper db(txc.DB, &*DsGroupSelector); - Context->ClearIndexes(); - return db.LoadIndexes(Self->GetPathId(), [&](const TInternalPathId /*pathId*/, const ui64 /*portionId*/, TIndexChunkLoadContext&& loadContext) { - Context->Add(std::move(loadContext)); - }); + return db.LoadIndexes( + Self->GetPathId(), [&](const TInternalPathId /*pathId*/, const ui64 /*portionId*/, TIndexChunkLoadContext&& loadContext) { + Context->Add(std::move(loadContext)); + }); } bool TGranuleIndexesReader::DoPrecharge(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { @@ -59,7 +59,8 @@ bool TGranuleFinishAccessorsLoading::DoExecute(NTabletFlatExecutor::TTransaction for (auto&& i : Self->GetPortions()) { auto it = constructors.find(i.first); AFL_VERIFY(it != constructors.end()); - auto accessor = TPortionAccessorConstructor::BuildForLoading(i.second, std::move(it->second.MutableRecords()), std::move(it->second.MutableIndexes())); + auto accessor = + TPortionAccessorConstructor::BuildForLoading(i.second, it->second.DetachBuildInfo(), std::move(it->second.MutableIndexes())); Self->GetDataAccessorsManager()->AddPortion(accessor); } return true; diff --git a/ydb/core/tx/columnshard/engines/storage/granule/stages.h b/ydb/core/tx/columnshard/engines/storage/granule/stages.h index 2dcac36dc4b0..d5d6b03d960a 100644 --- a/ydb/core/tx/columnshard/engines/storage/granule/stages.h +++ b/ydb/core/tx/columnshard/engines/storage/granule/stages.h @@ -11,39 +11,35 @@ namespace NKikimr::NOlap::NLoading { class TPortionDataAccessors { private: - YDB_ACCESSOR_DEF(std::vector, Records); + TColumnChunkLoadContextV2::TBuildInfo BuildInfo; std::vector Indexes; public: + TColumnChunkLoadContextV2::TBuildInfo DetachBuildInfo() { + return std::move(BuildInfo); + } + std::vector& MutableIndexes() { return Indexes; } - TPortionDataAccessors() = default; + TPortionDataAccessors(TColumnChunkLoadContextV2::TBuildInfo&& buildInfo) + : BuildInfo(std::move(buildInfo)) { + } }; class TPortionsLoadContext { private: THashMap Constructors; - TPortionDataAccessors& MutableConstructor(const ui64 portionId) { + TPortionDataAccessors& MutableConstructorVerified(const ui64 portionId) { auto it = Constructors.find(portionId); - if (it == Constructors.end()) { - it = Constructors.emplace(portionId, TPortionDataAccessors()).first; - } + AFL_VERIFY(it != Constructors.end()); return it->second; } public: - void ClearRecords() { - for (auto&& i : Constructors) { - i.second.MutableRecords().clear(); - } - } - - void ClearIndexes() { - for (auto&& i : Constructors) { - i.second.MutableIndexes().clear(); - } + void Clear() { + Constructors.clear(); } THashMap&& ExtractConstructors() { @@ -51,17 +47,11 @@ class TPortionsLoadContext { } void Add(TIndexChunkLoadContext&& chunk) { - auto& constructor = MutableConstructor(chunk.GetPortionId()); + auto& constructor = MutableConstructorVerified(chunk.GetPortionId()); constructor.MutableIndexes().emplace_back(std::move(chunk)); } - void Add(TColumnChunkLoadContextV1&& chunk) { - auto& constructor = MutableConstructor(chunk.GetPortionId()); - constructor.MutableRecords().emplace_back(std::move(chunk)); - } void Add(TColumnChunkLoadContextV2&& chunk) { - for (auto&& i : chunk.BuildRecordsV1()) { - Add(std::move(i)); - } + AFL_VERIFY(Constructors.emplace(chunk.GetPortionId(), chunk.CreateBuildInfo()).second); } }; @@ -132,6 +122,21 @@ class IGranuleTxReader: public ITxReader { } }; +class TGranuleStartAccessorsLoading: public IGranuleTxReader { +private: + using TBase = IGranuleTxReader; + virtual bool DoExecute(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + Context->Clear(); + return true; + } + virtual bool DoPrecharge(NTabletFlatExecutor::TTransactionContext& /*txc*/, const TActorContext& /*ctx*/) override { + return true; + } + +public: + using TBase::TBase; +}; + class TGranuleColumnsReader: public IGranuleTxReader { private: using TBase = IGranuleTxReader; diff --git a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h index 5496ac67faa6..3fc8b7d4f3e7 100644 --- a/ydb/core/tx/columnshard/normalizer/abstract/abstract.h +++ b/ydb/core/tx/columnshard/normalizer/abstract/abstract.h @@ -72,6 +72,7 @@ enum class ENormalizerSequentialId : ui32 { RestoreV2Chunks, CleanDeprecatedSnapshot, RestoreV0ChunksMeta, + CopyBlobIdsToV2, MAX }; diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index b44a0b57a50b..cba913769d73 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -32,7 +32,7 @@ class TNormalizerResult: public INormalizerChanges { "portion_id", portionInfo.GetPortionInfo().GetAddress().DebugString()); auto copy = portionInfo.GetPortionInfo().MakeCopy(); copy->SetRemoveSnapshot(TSnapshot(1, 1)); - db.WritePortion(*copy); + db.WritePortion({}, *copy); } if (BrokenPortions.size()) { NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.cpp b/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.cpp new file mode 100644 index 000000000000..fc034f009335 --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.cpp @@ -0,0 +1,115 @@ +#include "copy_blob_ids_to_v2.h" +#include "normalizer.h" + +#include +#include +#include +#include + +namespace NKikimr::NOlap::NCopyBlobIdsToV2 { + +class TBlobsWriting: public INormalizerChanges { +private: + THashMap> BlobsToWrite; + +public: + TBlobsWriting(THashMap>&& tasks) + : BlobsToWrite(std::move(tasks)) { + } + + virtual bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController&) const override { + using namespace NColumnShard; + NIceDb::TNiceDb db(txc.DB); + using IndexColumnsV2 = NColumnShard::Schema::IndexColumnsV2; + for (auto&& i : BlobsToWrite) { + NKikimrTxColumnShard::TIndexPortionBlobsInfo protoBlobs; + for (auto&& blobId : i.second) { + *protoBlobs.AddBlobIds() = blobId; + } + db.Table() + .Key(i.first.GetPathId().GetRawValue(), i.first.GetPortionId()) + .Update(NIceDb::TUpdate(protoBlobs.SerializeAsString())); + } + + return true; + } + + virtual ui64 GetSize() const override { + return BlobsToWrite.size(); + } +}; + +TConclusion> TNormalizer::DoInit( + const TNormalizationController& /*controller*/, NTabletFlatExecutor::TTransactionContext& txc) { + using namespace NColumnShard; + NIceDb::TNiceDb db(txc.DB); + + bool ready = true; + ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); + ready = ready & Schema::Precharge(db, txc.DB.GetScheme()); + if (!ready) { + return TConclusionStatus::Fail("Not ready"); + } + THashMap> portions0; + + { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return TConclusionStatus::Fail("Not ready"); + } + + while (!rowset.EndOfSet()) { + TPortionLoadContext portion(rowset); + AFL_VERIFY(portion.GetMetaProto().BlobIdsSize()); + std::vector blobIds; + for (auto&& i : portion.GetMetaProto().GetBlobIds()) { + blobIds.emplace_back(i); + } + AFL_VERIFY(portions0.emplace(portion.GetPortionId(), std::move(blobIds)).second); + + if (!rowset.Next()) { + return TConclusionStatus::Fail("Not ready"); + } + } + } + + std::vector tasks; + + { + auto rowset = db.Table().Select(); + if (!rowset.IsReady()) { + return TConclusionStatus::Fail("Not ready"); + } + THashMap> blobsByPortion; + while (!rowset.EndOfSet()) { + TColumnChunkLoadContextV2 chunk(rowset, DsGroupSelector); + auto it = portions0.find(chunk.GetPortionId()); + AFL_VERIFY(it != portions0.end()); + if (chunk.GetBlobIds().empty()) { + blobsByPortion.emplace(chunk.GetPortionAddress(), std::move(it->second)); + if (blobsByPortion.size() == 10000) { + tasks.emplace_back(std::make_shared(std::make_shared(std::move(blobsByPortion)))); + blobsByPortion.clear(); + } + } else { + AFL_VERIFY(it->second.size() == chunk.GetBlobIds().size()); + for (ui32 idx = 0; idx < it->second.size(); ++idx) { + AFL_VERIFY(it->second[idx] == chunk.GetBlobIds()[idx].GetLogoBlobId().AsBinaryString()); + } + } + portions0.erase(it); + + if (!rowset.Next()) { + return TConclusionStatus::Fail("Not ready"); + } + } + if (blobsByPortion.size()) { + tasks.emplace_back(std::make_shared(std::make_shared(std::move(blobsByPortion)))); + } + AFL_VERIFY(portions0.empty()); + } + + return tasks; +} + +} // namespace NKikimr::NOlap::NCopyBlobIdsToV2 diff --git a/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.h b/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.h new file mode 100644 index 000000000000..0470ddc2002c --- /dev/null +++ b/ydb/core/tx/columnshard/normalizer/portion/copy_blob_ids_to_v2.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr::NColumnShard { +class TTablesManager; +} + +namespace NKikimr::NOlap::NCopyBlobIdsToV2 { + +class TNormalizer: public TNormalizationController::INormalizerComponent { +private: + using TBase = TNormalizationController::INormalizerComponent; + +public: + static TString GetClassNameStatic() { + return ::ToString(ENormalizerSequentialId::CopyBlobIdsToV2); + } + + virtual std::optional DoGetEnumSequentialId() const override { + return ENormalizerSequentialId::CopyBlobIdsToV2; + } + + virtual TString GetClassName() const override { + return GetClassNameStatic(); + } + + class TNormalizerResult; + + static inline INormalizerComponent::TFactory::TRegistrator Registrator = + INormalizerComponent::TFactory::TRegistrator(GetClassNameStatic()); + +public: + TNormalizer(const TNormalizationController::TInitContext& info) + : TBase(info) + , DsGroupSelector(info.GetStorageInfo()) { + } + + virtual TConclusion> DoInit( + const TNormalizationController& controller, NTabletFlatExecutor::TTransactionContext& txc) override; + +private: + NColumnShard::TBlobGroupSelector DsGroupSelector; +}; +} // namespace NKikimr::NOlap::NCopyBlobIdsToV2 diff --git a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp index 8fea108e9f88..eaecec81f8c0 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.cpp @@ -1,8 +1,8 @@ -#include #include "leaked_blobs.h" #include #include +#include #include #include #include @@ -80,7 +80,7 @@ class TRemoveLeakedBlobsActor: public TActorBootstrapped> TLeakedBlobsNormalizer::DoInit( return TConclusionStatus::Fail("Not ready"); } - NColumnShard::TTablesManager tablesManager( - controller.GetStoragesManager(), controller.GetDataAccessorsManager(), std::make_shared(), std::make_shared(), TabletId); + NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), controller.GetDataAccessorsManager(), + std::make_shared(), std::make_shared(), TabletId); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TPortionsNormalizer")("error", "can't initialize tables manager"); @@ -209,24 +209,23 @@ TConclusionStatus TLeakedBlobsNormalizer::LoadPortionBlobIds( TDbWrapper wrapper(db.GetDatabase(), nullptr); if (Portions.empty()) { THashMap> portionsLocal; - if (!wrapper.LoadPortions({}, [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { - const TIndexInfo& indexInfo = - portion->GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); - AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, DsGroupSelector)); - const ui64 portionId = portion->GetPortionIdVerified(); - AFL_VERIFY(portionsLocal.emplace(portionId, std::move(portion)).second); - })) { + if (!wrapper.LoadPortions( + {}, [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = + portion->GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); + AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, DsGroupSelector)); + const ui64 portionId = portion->GetPortionIdVerified(); + AFL_VERIFY(portionsLocal.emplace(portionId, std::move(portion)).second); + })) { return TConclusionStatus::Fail("repeated read db"); } Portions = std::move(portionsLocal); } if (Records.empty()) { - THashMap> recordsLocal; + THashMap recordsLocal; if (!wrapper.LoadColumns(std::nullopt, [&](TColumnChunkLoadContextV2&& chunk) { const ui64 portionId = chunk.GetPortionId(); - for (auto&& i : chunk.BuildRecordsV1()) { - recordsLocal[portionId].emplace_back(std::move(i)); - } + AFL_VERIFY(recordsLocal.emplace(portionId, chunk.CreateBuildInfo()).second); })) { return TConclusionStatus::Fail("repeated read db"); } @@ -234,10 +233,11 @@ TConclusionStatus TLeakedBlobsNormalizer::LoadPortionBlobIds( } if (Indexes.empty()) { THashMap> indexesLocal; - if (!wrapper.LoadIndexes(std::nullopt, [&](const TInternalPathId /*pathId*/, const ui64 /*portionId*/, TIndexChunkLoadContext&& indexChunk) { - const ui64 portionId = indexChunk.GetPortionId(); - indexesLocal[portionId].emplace_back(std::move(indexChunk)); - })) { + if (!wrapper.LoadIndexes( + std::nullopt, [&](const TInternalPathId /*pathId*/, const ui64 /*portionId*/, TIndexChunkLoadContext&& indexChunk) { + const ui64 portionId = indexChunk.GetPortionId(); + indexesLocal[portionId].emplace_back(std::move(indexChunk)); + })) { return TConclusionStatus::Fail("repeated read db"); } Indexes = std::move(indexesLocal); diff --git a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.h b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.h index a81d8db23112..c62c6a40ac13 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.h +++ b/ydb/core/tx/columnshard/normalizer/portion/leaked_blobs.h @@ -2,6 +2,7 @@ #include "normalizer.h" +#include #include #include @@ -9,6 +10,7 @@ namespace NKikimr::NOlap { class TLeakedBlobsNormalizer: public TNormalizationController::INormalizerComponent { private: using TBase = TNormalizationController::INormalizerComponent; + public: static TString GetClassNameStatic() { return "LeakedBlobsNormalizer"; @@ -43,7 +45,7 @@ class TLeakedBlobsNormalizer: public TNormalizationController::INormalizerCompon TActorId TRemoveLeakedBlobsActorId; NColumnShard::TBlobGroupSelector DsGroupSelector; THashMap> Portions; - THashMap> Records; + THashMap Records; THashMap> Indexes; THashSet BlobsToDelete; }; diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp index 04b7ad481fe5..c3b6f5689d93 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp @@ -95,13 +95,14 @@ TConclusion> TPortionsNormalizerBase::DoInit( TConclusionStatus TPortionsNormalizerBase::InitPortions( const NColumnShard::TTablesManager& tablesManager, NIceDb::TNiceDb& db, THashMap& constructors) { TDbWrapper wrapper(db.GetDatabase(), nullptr); - if (!wrapper.LoadPortions({}, [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { - const TIndexInfo& indexInfo = - portion->GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); - AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, DsGroupSelector)); - const ui64 portionId = portion->GetPortionIdVerified(); - AFL_VERIFY(constructors.emplace(portionId, TPortionAccessorConstructor(std::move(portion))).second); - })) { + if (!wrapper.LoadPortions( + {}, [&](std::unique_ptr&& portion, const NKikimrTxColumnShard::TIndexPortionMeta& metaProto) { + const TIndexInfo& indexInfo = + portion->GetSchema(tablesManager.GetPrimaryIndexAsVerified().GetVersionedIndex())->GetIndexInfo(); + AFL_VERIFY(portion->MutableMeta().LoadMetadata(metaProto, indexInfo, DsGroupSelector)); + const ui64 portionId = portion->GetPortionIdVerified(); + AFL_VERIFY(constructors.emplace(portionId, TPortionAccessorConstructor(std::move(portion))).second); + })) { return TConclusionStatus::Fail("repeated read db"); } return TConclusionStatus::Success(); @@ -116,20 +117,11 @@ TConclusionStatus TPortionsNormalizerBase::InitColumns( return TConclusionStatus::Fail("Not ready"); } - auto initPortion = [&](TColumnChunkLoadContextV1&& loadContext) { - if (!columnsFilter.empty() && !columnsFilter.contains(loadContext.GetAddress().GetColumnId())) { - return; - } - auto it = portions.find(loadContext.GetPortionId()); - AFL_VERIFY(it != portions.end()); - it->second.LoadRecord(std::move(loadContext)); - }; - while (!rowset.EndOfSet()) { - NOlap::TColumnChunkLoadContextV2 chunkLoadContext(rowset); - for (auto&& i : chunkLoadContext.BuildRecordsV1()) { - initPortion(std::move(i)); - } + NOlap::TColumnChunkLoadContextV2 chunkLoadContext(rowset, DsGroupSelector); + auto it = portions.find(chunkLoadContext.GetPortionId()); + AFL_VERIFY(it != portions.end()); + it->second.AddBuildInfo(chunkLoadContext.CreateBuildInfo()); if (!rowset.Next()) { return TConclusionStatus::Fail("Not ready"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/ya.make b/ydb/core/tx/columnshard/normalizer/portion/ya.make index 79196335804c..c28da5e4268b 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/ya.make +++ b/ydb/core/tx/columnshard/normalizer/portion/ya.make @@ -18,6 +18,7 @@ SRCS( GLOBAL clean_unused_tables_template.cpp GLOBAL clean_ttl_preset_setting_info.cpp GLOBAL clean_ttl_preset_setting_version_info.cpp + GLOBAL copy_blob_ids_to_v2.cpp ) PEERDIR( diff --git a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp index e1bfabe4514c..aa8f2753ca00 100644 --- a/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp +++ b/ydb/core/tx/columnshard/ut_rw/ut_normalizer.cpp @@ -47,7 +47,7 @@ class TNormalizerChecker { } }; -class TSchemaVersionsCleaner : public NYDBTest::ILocalDBModifier { +class TSchemaVersionsCleaner: public NYDBTest::ILocalDBModifier { public: virtual void Apply(NTabletFlatExecutor::TTransactionContext& txc) const override { using namespace NColumnShard; @@ -75,7 +75,7 @@ class TSchemaVersionsCleaner : public NYDBTest::ILocalDBModifier { } }; -class TPortionsCleaner : public NYDBTest::ILocalDBModifier { +class TPortionsCleaner: public NYDBTest::ILocalDBModifier { public: virtual void Apply(NTabletFlatExecutor::TTransactionContext& txc) const override { using namespace NColumnShard; @@ -87,8 +87,8 @@ class TPortionsCleaner : public NYDBTest::ILocalDBModifier { UNIT_ASSERT(rowset.IsReady()); while (!rowset.EndOfSet()) { - NOlap::TPortionAddress addr( - TInternalPathId::FromRawValue(rowset.GetValue()), rowset.GetValue()); + NOlap::TPortionAddress addr(TInternalPathId::FromRawValue(rowset.GetValue()), + rowset.GetValue()); portions.emplace_back(addr); UNIT_ASSERT(rowset.Next()); } @@ -111,8 +111,8 @@ class TEmptyPortionsCleaner: public NYDBTest::ILocalDBModifier { metaProto.SetDeletionsCount(0); metaProto.SetIsInserted(true); - const auto schema = std::make_shared( - arrow::FieldVector({ std::make_shared("key1", arrow::uint64()), std::make_shared("key2", arrow::uint64()) })); + const auto schema = std::make_shared(arrow::FieldVector( + { std::make_shared("key1", arrow::uint64()), std::make_shared("key2", arrow::uint64()) })); auto batch = NArrow::MakeEmptyBatch(schema, 1); NArrow::TFirstLastSpecialKeys keys(batch); metaProto.SetPrimaryKeyBorders(keys.SerializePayloadToString()); @@ -209,7 +209,7 @@ class TPrepareLocalDBController: public NKikimr::NYDBTest::NColumnShard::TContro } }; -class TTrashUnusedInjector : public NYDBTest::ILocalDBModifier { +class TTrashUnusedInjector: public NYDBTest::ILocalDBModifier { public: void Apply(NTabletFlatExecutor::TTransactionContext& txc) const override { using namespace NColumnShard; @@ -218,9 +218,7 @@ class TTrashUnusedInjector : public NYDBTest::ILocalDBModifier { if (db.HaveTable()) { for (size_t i = 0; i < 100; ++i) { - db.Table() - .Key(1 + i, 2 + i, 3 + i, 4 + i, 5 + i, 6 + i, 7 + i) - .Update(); + db.Table().Key(1 + i, 2 + i, 3 + i, 4 + i, 5 + i, 6 + i, 7 + i).Update(); } } } @@ -236,8 +234,8 @@ Y_UNIT_TEST_SUITE(Normalizers) { TTester::Setup(runtime); runtime.GetAppData().ColumnShardConfig.SetColumnChunksV0Usage(false); - checker.CorrectConfigurationOnStart(runtime.GetAppData().ColumnShardConfig); - checker.CorrectFeatureFlagsOnStart(runtime.GetAppData().FeatureFlags); + checker.CorrectConfigurationOnStart(runtime.GetAppData().ColumnShardConfig); + checker.CorrectFeatureFlagsOnStart(runtime.GetAppData().FeatureFlags); const ui64 tableId = 1; const std::vector schema = { NArrow::NTest::TTestColumn("key1", TTypeInfo(NTypeIds::Uint64)), @@ -277,7 +275,7 @@ Y_UNIT_TEST_SUITE(Normalizers) { virtual ui64 RecordsCountAfterReboot(const ui64 /*initialRecodsCount*/) const override { return 0; } - virtual void CorrectFeatureFlagsOnStart(TFeatureFlags & featuresFlags) const override{ + virtual void CorrectFeatureFlagsOnStart(TFeatureFlags& featuresFlags) const override { featuresFlags.SetEnableWritePortionsOnInsert(true); } virtual void CorrectConfigurationOnStart(NKikimrConfig::TColumnShardConfig& columnShardConfig) const override { @@ -323,7 +321,6 @@ Y_UNIT_TEST_SUITE(Normalizers) { TestNormalizerImpl(TLocalNormalizerChecker()); } - Y_UNIT_TEST(EmptyTablesNormalizer) { class TLocalNormalizerChecker: public TNormalizerChecker { public: From 54e5eb9f5686251dec008db7dc9c6e309bb6b139 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Fri, 27 Jun 2025 17:53:12 +0300 Subject: [PATCH 42/62] fix problem portions processing (#20297) Co-authored-by: ivanmorozov333 --- ydb/core/kqp/ut/olap/optimizer_ut.cpp | 16 +++++++++++++++- .../optimizer/lcbuckets/planner/optimizer.h | 5 +++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ydb/core/kqp/ut/olap/optimizer_ut.cpp b/ydb/core/kqp/ut/olap/optimizer_ut.cpp index b4d7ac2ac458..3e7258a35559 100644 --- a/ydb/core/kqp/ut/olap/optimizer_ut.cpp +++ b/ydb/core/kqp/ut/olap/optimizer_ut.cpp @@ -147,7 +147,8 @@ Y_UNIT_TEST_SUITE(KqpOlapOptimizer) { UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString()); auto rows = CollectRows(it); for (auto&& i : rows) { - Cerr << GetUint64(i.at("LEVEL")) << "/" << GetUint64(i.at("RECORDS_COUNT_DEFAULT")) << "/" << GetUint64(i.at("RECORDS_COUNT_SLICE")) << Endl; + Cerr << GetUint64(i.at("LEVEL")) << "/" << GetUint64(i.at("RECORDS_COUNT_DEFAULT")) << "/" + << GetUint64(i.at("RECORDS_COUNT_SLICE")) << Endl; } AFL_VERIFY(0 == GetUint64(rows[0].at("LEVEL"))); AFL_VERIFY(GetUint64(rows[0].at("RECORDS_COUNT_DEFAULT")) == 0); @@ -227,7 +228,20 @@ Y_UNIT_TEST_SUITE(KqpOlapOptimizer) { ++levelIdx; } } + + { + auto alterQuery = + TStringBuilder() << + R"(ALTER OBJECT `/Root/olapStore` (TYPE TABLESTORE) SET (ACTION=UPSERT_OPTIONS, `COMPACTION_PLANNER.CLASS_NAME`=`lc-buckets`, `COMPACTION_PLANNER.FEATURES`=` + {"levels" : [{"class_name" : "Zero", "expected_blobs_size" : 20000, "portions_size_limit" : 100000, "portions_count_available" : 1}, + {"class_name" : "Zero"}]}`); + )"; + auto session = tableClient.CreateSession().GetValueSync().GetSession(); + auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); + } } + Y_UNIT_TEST(OptimizationByTime) { auto settings = TKikimrSettings().SetWithSampleTables(false); TKikimrRunner kikimr(settings); diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h index 4dd80a0e1489..8f9986096dd1 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lcbuckets/planner/optimizer.h @@ -86,9 +86,10 @@ class TOptimizerPlanner: public IOptimizerPlanner { continue; } PortionsInfo->AddPortion(i); - addPortionsByLevels[i->GetMeta().GetCompactionLevel()].emplace_back(i); - if (i->GetCompactionLevel() && i->GetCompactionLevel() >= Levels.size()) { + if (i->GetCompactionLevel() >= Levels.size()) { problemPortions.emplace_back(i); + } else { + addPortionsByLevels[i->GetMeta().GetCompactionLevel()].emplace_back(i); } } for (ui32 i = 0; i < Levels.size(); ++i) { From 6754d8d0ef834ace7650ed55054cf5059f746793 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Mon, 30 Jun 2025 10:00:11 +0300 Subject: [PATCH 43/62] fix signals and separate cache by source to kill processing (#20330) Co-authored-by: ivanmorozov333 --- ydb/core/protos/config.proto | 3 +- .../transaction/tx_blobs_written.cpp | 2 + ydb/core/tx/columnshard/columnshard.cpp | 14 +- .../tx/columnshard/columnshard__write.cpp | 6 +- ydb/core/tx/columnshard/columnshard_impl.h | 1 - .../columnshard/columnshard_private_events.h | 2 - .../data_accessor/cache_policy/policy.cpp | 18 +- .../data_accessor/cache_policy/policy.h | 5 + .../tx/columnshard/data_accessor/manager.h | 10 +- .../engines/changes/compaction/merger.cpp | 2 + .../engines/reader/transaction/tx_scan.cpp | 23 +- .../engines/writer/buffer/actor.cpp | 59 --- .../columnshard/engines/writer/buffer/actor.h | 36 -- .../engines/writer/buffer/events.h | 28 -- .../columnshard/engines/writer/buffer/ya.make | 1 - .../columnshard/operations/common/context.h | 5 +- ydb/core/tx/columnshard/operations/events.cpp | 1 - ydb/core/tx/columnshard/operations/events.h | 1 - .../operations/slice_builder/builder.cpp | 3 + .../operations/slice_builder/builder.h | 2 +- .../operations/slice_builder/pack_builder.cpp | 8 + .../operations/slice_builder/pack_builder.h | 2 +- .../tx/columnshard/tablet/write_queue.cpp | 3 +- .../tx/data_events/common/signals_flow.cpp | 16 +- ydb/core/tx/data_events/common/signals_flow.h | 36 +- ydb/core/tx/data_events/write_data.cpp | 13 +- ydb/core/tx/data_events/write_data.h | 17 +- ydb/core/tx/general_cache/service/counters.h | 33 +- ydb/core/tx/general_cache/service/manager.h | 387 +++++++++++++----- ydb/core/tx/general_cache/service/service.h | 18 +- ydb/core/tx/general_cache/source/abstract.h | 14 +- ydb/core/tx/general_cache/source/events.h | 22 +- ydb/core/tx/general_cache/usage/config.cpp | 10 +- ydb/core/tx/general_cache/usage/config.h | 3 +- ydb/core/tx/general_cache/usage/events.h | 16 + ydb/core/tx/general_cache/usage/service.h | 12 +- 36 files changed, 471 insertions(+), 361 deletions(-) delete mode 100644 ydb/core/tx/columnshard/engines/writer/buffer/actor.cpp delete mode 100644 ydb/core/tx/columnshard/engines/writer/buffer/actor.h diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 119920a067c9..359a55b9258c 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -2263,7 +2263,8 @@ message TLabel { message TGeneralCacheConfig { optional uint64 MemoryLimit = 1; - optional uint64 DirectInflightLimit = 2; + optional uint64 DirectInflightSourceLimit = 2; + optional uint64 DirectInflightGlobalLimit = 3; } message TAppConfig { diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp index 9c4be6243911..76bfa8f1aa81 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp @@ -135,6 +135,7 @@ void TTxBlobsWritingFinished::DoComplete(const TActorContext& ctx) { } Self->Counters.GetCSCounters().OnWriteTxComplete(now - writeMeta.GetWriteStartInstant()); Self->Counters.GetCSCounters().OnSuccessWriteResponse(); + writeMeta.OnStage(NEvWrite::EWriteStage::Finished); } Self->SetupCompaction(pathIds); } @@ -171,6 +172,7 @@ void TTxBlobsWritingFailed::DoComplete(const TActorContext& ctx) { } for (auto&& wResult : Pack.GetWriteResults()) { const auto& writeMeta = wResult.GetWriteMeta(); + writeMeta.OnStage(NEvWrite::EWriteStage::Aborted); auto op = Self->GetOperationsManager().GetOperationVerified((TOperationWriteId)writeMeta.GetWriteId()); Self->OperationsManager->AbortTransactionOnComplete(*Self, op->GetLockId()); } diff --git a/ydb/core/tx/columnshard/columnshard.cpp b/ydb/core/tx/columnshard/columnshard.cpp index 4d2c81d204a2..b38682554c96 100644 --- a/ydb/core/tx/columnshard/columnshard.cpp +++ b/ydb/core/tx/columnshard/columnshard.cpp @@ -4,7 +4,6 @@ #include "blobs_reader/actor.h" #include "counters/aggregation/table_stats.h" #include "engines/column_engine_logs.h" -#include "engines/writer/buffer/actor.h" #include "engines/writer/buffer/actor2.h" #include "hooks/abstract/abstract.h" #include "resource_subscriber/actor.h" @@ -32,8 +31,8 @@ void TColumnShard::CleanupActors(const TActorContext& ctx) { } InFlightReadsTracker.Stop(this); ctx.Send(ResourceSubscribeActor, new TEvents::TEvPoisonPill); - ctx.Send(BufferizationInsertionWriteActorId, new TEvents::TEvPoisonPill); ctx.Send(BufferizationPortionsWriteActorId, new TEvents::TEvPoisonPill); + NGeneralCache::TServiceOperator::KillSource(SelfId()); if (!!OperationsManager) { OperationsManager->StopWriting(); } @@ -120,7 +119,6 @@ void TColumnShard::OnActivateExecutor(const TActorContext& ctx) { Limits.RegisterControls(icb); Settings.RegisterControls(icb); ResourceSubscribeActor = ctx.Register(new NOlap::NResourceBroker::NSubscribe::TActor(TabletID(), SelfId())); - BufferizationInsertionWriteActorId = ctx.Register(new NColumnShard::NWriting::TActor(TabletID(), SelfId())); BufferizationPortionsWriteActorId = ctx.Register(new NOlap::NWritingPortions::TActor(TabletID(), SelfId())); DataAccessorsManager = std::make_shared(SelfId()); NormalizerController.SetDataAccessorsManager(DataAccessorsManager); @@ -296,7 +294,7 @@ void TColumnShard::UpdateIndexCounters() { auto insertedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_INSERTED_PORTIONS, insertedStats.GetCount()); -// counters->SetCounter(COUNTER_INSERTED_BLOBS, insertedStats.GetBlobs()); + // counters->SetCounter(COUNTER_INSERTED_BLOBS, insertedStats.GetBlobs()); counters->SetCounter(COUNTER_INSERTED_ROWS, insertedStats.GetRecordsCount()); counters->SetCounter(COUNTER_INSERTED_BYTES, insertedStats.GetBlobBytes()); counters->SetCounter(COUNTER_INSERTED_RAW_BYTES, insertedStats.GetRawBytes()); @@ -304,7 +302,7 @@ void TColumnShard::UpdateIndexCounters() { auto compactedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_COMPACTED_PORTIONS, compactedStats.GetCount()); -// counters->SetCounter(COUNTER_COMPACTED_BLOBS, compactedStats.GetBlobs()); + // counters->SetCounter(COUNTER_COMPACTED_BLOBS, compactedStats.GetBlobs()); counters->SetCounter(COUNTER_COMPACTED_ROWS, compactedStats.GetRecordsCount()); counters->SetCounter(COUNTER_COMPACTED_BYTES, compactedStats.GetBlobBytes()); counters->SetCounter(COUNTER_COMPACTED_RAW_BYTES, compactedStats.GetRawBytes()); @@ -312,7 +310,7 @@ void TColumnShard::UpdateIndexCounters() { auto splitCompactedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_PORTIONS, splitCompactedStats.GetCount()); -// counters->SetCounter(COUNTER_SPLIT_COMPACTED_BLOBS, splitCompactedStats.GetBlobs()); + // counters->SetCounter(COUNTER_SPLIT_COMPACTED_BLOBS, splitCompactedStats.GetBlobs()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_ROWS, splitCompactedStats.GetRecordsCount()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_BYTES, splitCompactedStats.GetBlobBytes()); counters->SetCounter(COUNTER_SPLIT_COMPACTED_RAW_BYTES, splitCompactedStats.GetRawBytes()); @@ -320,7 +318,7 @@ void TColumnShard::UpdateIndexCounters() { auto inactiveStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_INACTIVE_PORTIONS, inactiveStats.GetCount()); -// counters->SetCounter(COUNTER_INACTIVE_BLOBS, inactiveStats.GetBlobs()); + // counters->SetCounter(COUNTER_INACTIVE_BLOBS, inactiveStats.GetBlobs()); counters->SetCounter(COUNTER_INACTIVE_ROWS, inactiveStats.GetRecordsCount()); counters->SetCounter(COUNTER_INACTIVE_BYTES, inactiveStats.GetBlobBytes()); counters->SetCounter(COUNTER_INACTIVE_RAW_BYTES, inactiveStats.GetRawBytes()); @@ -328,7 +326,7 @@ void TColumnShard::UpdateIndexCounters() { auto evictedStats = Counters.GetPortionIndexCounters()->GetTotalStats(TPortionIndexStats::TPortionsByType()); counters->SetCounter(COUNTER_EVICTED_PORTIONS, evictedStats.GetCount()); -// counters->SetCounter(COUNTER_EVICTED_BLOBS, evictedStats.GetBlobs()); + // counters->SetCounter(COUNTER_EVICTED_BLOBS, evictedStats.GetBlobs()); counters->SetCounter(COUNTER_EVICTED_ROWS, evictedStats.GetRecordsCount()); counters->SetCounter(COUNTER_EVICTED_BYTES, evictedStats.GetBlobBytes()); counters->SetCounter(COUNTER_EVICTED_RAW_BYTES, evictedStats.GetRawBytes()); diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index fce4c14e3a24..081e05a49973 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -104,7 +104,7 @@ void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& e for (auto&& i : writtenData.GetWriteResults()) { AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", i.GetDataSize())("event", "data_write_finished")( "writing_id", i.GetWriteMeta().GetId()); - i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::Finished); + i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::SuccessWritingToLocalDB); Counters.OnWritePutBlobsSuccess(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); Counters.GetWritesMonitor()->OnFinishWrite(i.GetDataSize(), 1); } @@ -112,12 +112,12 @@ void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& e } else { const TMonotonic now = TMonotonic::Now(); for (auto&& i : writtenData.GetWriteResults()) { + i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::FailWritingToLocalDB); Counters.OnWritePutBlobsFailed(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); Counters.GetCSCounters().OnWritePutBlobsFail(now - i.GetWriteMeta().GetWriteStartInstant()); AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", i.GetDataSize())("event", "data_write_error")( "writing_id", i.GetWriteMeta().GetId())("reason", i.GetErrorMessage()); Counters.GetWritesMonitor()->OnFinishWrite(i.GetDataSize(), 1); - i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::Finished); } Execute(new TTxBlobsWritingFailed(this, std::move(writtenData)), ctx); @@ -137,7 +137,7 @@ void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActo for (auto&& aggr : baseAggregations) { const auto& writeMeta = aggr->GetWriteMeta(); - aggr->MutableWriteMeta().OnStage(NEvWrite::EWriteStage::Finished); + aggr->MutableWriteMeta().OnStage(NEvWrite::EWriteStage::Aborted); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "blobs_write_finished")("writing_size", aggr->GetSize())( "writing_id", writeMeta.GetId())("status", putResult.GetPutStatus()); Counters.GetWritesMonitor()->OnFinishWrite(aggr->GetSize(), 1); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 1f510242fa97..a98fbb8f6bd5 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -504,7 +504,6 @@ class TColumnShard: public TActor, public NTabletFlatExecutor::TTa TInstant LastStatsReport; TActorId ResourceSubscribeActor; - TActorId BufferizationInsertionWriteActorId; TActorId BufferizationPortionsWriteActorId; NOlap::NDataAccessorControl::TDataAccessorsManagerContainer DataAccessorsManager; diff --git a/ydb/core/tx/columnshard/columnshard_private_events.h b/ydb/core/tx/columnshard/columnshard_private_events.h index 21db6025e4df..c9ccc1eef082 100644 --- a/ydb/core/tx/columnshard/columnshard_private_events.h +++ b/ydb/core/tx/columnshard/columnshard_private_events.h @@ -47,8 +47,6 @@ struct TEvPrivate { EvStartResourceUsageTask, EvNormalizerResult, - EvWritingAddDataToBuffer, - EvWritingFlushBuffer, EvWritingPortionsAddDataToBuffer, EvWritingPortionsFlushBuffer, diff --git a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp index 55134427e8ca..6468fc9788d1 100644 --- a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp +++ b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.cpp @@ -22,7 +22,7 @@ TPortionsMetadataCachePolicy::BuildObjectsProcessor(const NActors::TActorId& ser AFL_VERIFY(RequestedAddresses.erase(address)); objects.emplace(address, std::move(i)); } - Callback->OnReceiveData(std::move(objects), std::move(RequestedAddresses), {}); + Callback->OnReceiveData(OwnerActorId, std::move(objects), std::move(RequestedAddresses), {}); } public: @@ -39,6 +39,7 @@ TPortionsMetadataCachePolicy::BuildObjectsProcessor(const NActors::TActorId& ser private: using TAddress = TGlobalPortionAddress; using TObject = TPortionDataAccessor; + using TSourceId = NActors::TActorId; using EConsumer = TPortionsMetadataCachePolicy::EConsumer; using TSelf = NKikimr::NGeneralCache::NSource::IObjectsProcessor; @@ -63,8 +64,8 @@ TPortionsMetadataCachePolicy::BuildObjectsProcessor(const NActors::TActorId& ser } }; - virtual void DoAskData( - const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const override { + virtual void DoAskData(const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr, + const ui64 cookie) const override { THashMap requests; for (auto&& [c, addresses] : objectAddressesByConsumer) { for (auto&& a : addresses) { @@ -72,16 +73,17 @@ TPortionsMetadataCachePolicy::BuildObjectsProcessor(const NActors::TActorId& ser } } for (auto&& i : requests) { - NActors::TActivationContext::Send( - i.first, std::make_unique(i.second.ExtractRequest(), - std::make_shared(i.first, selfPtr, i.second.ExtractRequestedAddresses()))); + NActors::TActivationContext::Send(i.first, + std::make_unique( + i.second.ExtractRequest(), std::make_shared(i.first, selfPtr, i.second.ExtractRequestedAddresses())), + 0, cookie); } } - virtual void DoOnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + virtual void DoOnReceiveData(const TSourceId sourceId, THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errors) const override { NActors::TActivationContext::Send( ServiceActorId, std::make_unique::TEvObjectsInfo>( - std::move(objectAddresses), std::move(removedAddresses), std::move(errors))); + sourceId, std::move(objectAddresses), std::move(removedAddresses), std::move(errors))); } public: diff --git a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h index f9655a084d3e..cac4b4ecebde 100644 --- a/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h +++ b/ydb/core/tx/columnshard/data_accessor/cache_policy/policy.h @@ -44,8 +44,13 @@ class TPortionsMetadataCachePolicy { public: using TAddress = TGlobalPortionAddress; using TObject = TPortionDataAccessor; + using TSourceId = NActors::TActorId; using EConsumer = NOlap::NBlobOperations::EConsumer; + static TSourceId GetSourceId(const TAddress& address) { + return address.GetTabletActorId(); + } + static EConsumer DefaultConsumer() { return EConsumer::UNDEFINED; } diff --git a/ydb/core/tx/columnshard/data_accessor/manager.h b/ydb/core/tx/columnshard/data_accessor/manager.h index 21a816c61401..5f53623a8cd0 100644 --- a/ydb/core/tx/columnshard/data_accessor/manager.h +++ b/ydb/core/tx/columnshard/data_accessor/manager.h @@ -66,13 +66,15 @@ class TActorAccessorsManager: public IDataAccessorsManager { THashSet&& removedAddresses, THashMap&& errorAddresses) const override { AFL_VERIFY(removedAddresses.empty()); - AFL_VERIFY(errorAddresses.empty()); THashMap objects; for (auto&& i : objectAddresses) { objects.emplace(i.first.GetPortionId(), std::move(i.second)); } TDataAccessorsResult result; result.AddData(std::move(objects)); + for (auto&& i : errorAddresses) { + result.AddError(i.first.GetPathId(), i.second); + } AccessorsCallback->OnResult(RequestId, std::move(result)); } @@ -91,13 +93,15 @@ class TActorAccessorsManager: public IDataAccessorsManager { THashMap add; THashSet remove; add.emplace(NGeneralCache::TGlobalPortionAddress(GetTabletActorId(), accessor.GetPortionInfo().GetAddress()), accessor); - NKikimr::NGeneralCache::TServiceOperator::ModifyObjects(std::move(add), std::move(remove)); + NKikimr::NGeneralCache::TServiceOperator::ModifyObjects( + GetTabletActorId(), std::move(add), std::move(remove)); } virtual void DoRemovePortion(const TPortionInfo::TConstPtr& portion) override { THashMap add; THashSet remove; remove.emplace(NGeneralCache::TGlobalPortionAddress(GetTabletActorId(), portion->GetAddress())); - NKikimr::NGeneralCache::TServiceOperator::ModifyObjects(std::move(add), std::move(remove)); + NKikimr::NGeneralCache::TServiceOperator::ModifyObjects( + GetTabletActorId(), std::move(add), std::move(remove)); } public: diff --git a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp index 46e8fdccb4a1..7771e9bc612a 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction/merger.cpp @@ -187,6 +187,8 @@ class TSplittedBatch { } } if (!colStatsOpt) { + AFL_WARN(NKikimrServices::TX_COLUMNSHARD_COMPACTION)("event", "incorrect_case_stat")("stat", Stats->DebugString())( + "column_id", c)("schema", resultFiltered->DebugString()); chunks = NArrow::NSplitter::TSimilarPacker::SplitWithExpected(p, settings.GetExpectedRecordsCountOnPage()); } else { chunks = colStatsOpt->SplitRecords( diff --git a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp index 478d892ad556..b40ae77adfe2 100644 --- a/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp +++ b/ydb/core/tx/columnshard/engines/reader/transaction/tx_scan.cpp @@ -38,13 +38,12 @@ void TTxScan::Complete(const TActorContext& ctx) { if (snapshot.IsZero()) { snapshot = Self->GetLastTxSnapshot(); } - const TReadMetadataBase::ESorting sorting = - [&]() { - if (request.HasReverse()) { - return request.GetReverse() ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC; - } else { - return TReadMetadataBase::ESorting::NONE; - } + const TReadMetadataBase::ESorting sorting = [&]() { + if (request.HasReverse()) { + return request.GetReverse() ? TReadMetadataBase::ESorting::DESC : TReadMetadataBase::ESorting::ASC; + } else { + return TReadMetadataBase::ESorting::NONE; + } }(); TScannerConstructorContext context(snapshot, request.HasItemsLimit() ? request.GetItemsLimit() : 0, sorting); @@ -74,12 +73,11 @@ void TTxScan::Complete(const TActorContext& ctx) { const auto& schemeShardLocalPathId = NColumnShard::TSchemeShardLocalPathId::FromProto(request); const auto& internalPathId = Self->TablesManager.ResolveInternalPathId(schemeShardLocalPathId); - read.PathId = NColumnShard::TUnifiedPathId{internalPathId ? *internalPathId : TInternalPathId{}, schemeShardLocalPathId}; + read.PathId = NColumnShard::TUnifiedPathId{ internalPathId ? *internalPathId : TInternalPathId{}, schemeShardLocalPathId }; read.ReadNothing = !Self->TablesManager.HasTable(read.PathId.InternalPathId); read.TableName = table; - const TString defaultReader = - [&]() { + const TString defaultReader = [&]() { const TString defGlobal = AppDataVerified().ColumnShardConfig.GetReaderClassName() ? AppDataVerified().ColumnShardConfig.GetReaderClassName() : "PLAIN"; if (Self->HasIndex()) { @@ -164,9 +162,8 @@ void TTxScan::Complete(const TActorContext& ctx) { AFL_VERIFY(shardingPolicy.DeserializeFromProto(request.GetComputeShardingPolicy())); auto scanActorId = ctx.Register(new TColumnShardScan(Self->SelfId(), scanComputeActor, Self->GetStoragesManager(), - Self->DataAccessorsManager.GetObjectPtrVerified(), shardingPolicy, scanId, - txId, scanGen, requestCookie, Self->TabletID(), timeout, readMetadataRange, dataFormat, Self->Counters.GetScanCounters(), - cpuLimits)); + Self->DataAccessorsManager.GetObjectPtrVerified(), shardingPolicy, scanId, txId, scanGen, requestCookie, Self->TabletID(), timeout, + readMetadataRange, dataFormat, Self->Counters.GetScanCounters(), cpuLimits)); Self->InFlightReadsTracker.AddScanActorId(requestCookie, scanActorId); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("event", "TTxScan started")("actor_id", scanActorId)("trace_detailed", detailedInfo); diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/actor.cpp b/ydb/core/tx/columnshard/engines/writer/buffer/actor.cpp deleted file mode 100644 index eeb6242103d1..000000000000 --- a/ydb/core/tx/columnshard/engines/writer/buffer/actor.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include "actor.h" -#include -#include - -namespace NKikimr::NColumnShard::NWriting { - -TActor::TActor(ui64 tabletId, const TActorId& parent) - : TabletId(tabletId) - , ParentActorId(parent) -{ - -} - -void TActor::Bootstrap() { - Become(&TThis::StateWait); - Schedule(FlushDuration, new TEvFlushBuffer); - FlushDuration = TDuration::MilliSeconds(AppDataVerified().ColumnShardConfig.GetWritingBufferDurationMs()); -} - -void TActor::Flush() { - if (Aggregations.size()) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "flush_writing")("size", SumSize)("count", Aggregations.size()); - auto action = Aggregations.front()->GetBlobsAction(); - auto writeController = std::make_shared(ParentActorId, action, std::move(Aggregations)); - if (action->NeedDraftTransaction()) { - TActorContext::AsActorContext().Send(ParentActorId, std::make_unique(writeController)); - } else { - TActorContext::AsActorContext().Register(NColumnShard::CreateWriteActor(TabletId, writeController, TInstant::Max())); - } - Aggregations.clear(); - SumSize = 0; - } else { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "skip_flush_writing"); - } -} - -void TActor::Handle(TEvFlushBuffer::TPtr& /*ev*/) { - FlushDuration = TDuration::MilliSeconds(AppDataVerified().ColumnShardConfig.GetWritingBufferDurationMs()); - Flush(); - if (!FlushDuration) { - Schedule(TDuration::MilliSeconds(500), new TEvFlushBuffer); - } else { - Schedule(FlushDuration, new TEvFlushBuffer); - } -} - -void TActor::Handle(TEvAddInsertedDataToBuffer::TPtr& ev) { - auto* evBase = ev->Get(); - AFL_VERIFY(evBase->GetWriteData()->GetBlobsAction()->GetStorageId() == NOlap::IStoragesManager::DefaultStorageId); - SumSize += evBase->GetWriteData()->GetSize(); - Aggregations.emplace_back( - std::make_shared(*evBase->GetWriteData(), std::move(evBase->MutableBlobsToWrite()), evBase->GetRecordBatch())); - if (SumSize > 4 * 1024 * 1024 || Aggregations.size() > 750 || !FlushDuration) { - Flush(); - } -} - - -} diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/actor.h b/ydb/core/tx/columnshard/engines/writer/buffer/actor.h deleted file mode 100644 index a190b6ec2829..000000000000 --- a/ydb/core/tx/columnshard/engines/writer/buffer/actor.h +++ /dev/null @@ -1,36 +0,0 @@ -#pragma once -#include "events.h" -#include -#include - -namespace NKikimr::NColumnShard::NWriting { - -class TActor: public TActorBootstrapped { -private: - std::vector> Aggregations; - const ui64 TabletId; - NActors::TActorId ParentActorId; - TDuration FlushDuration = TDuration::Zero(); - ui64 SumSize = 0; - void Flush(); -public: - TActor(ui64 tabletId, const TActorId& parent); - ~TActor() = default; - - void Handle(TEvAddInsertedDataToBuffer::TPtr& ev); - void Handle(TEvFlushBuffer::TPtr& ev); - void Bootstrap(); - - STFUNC(StateWait) { - TLogContextGuard gLogging(NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletId)("parent", ParentActorId)); - switch (ev->GetTypeRewrite()) { - cFunc(NActors::TEvents::TEvPoison::EventType, PassAway); - hFunc(TEvAddInsertedDataToBuffer, Handle); - hFunc(TEvFlushBuffer, Handle); - default: - AFL_VERIFY(false)("ev_type", ev->GetTypeName()); - } - } -}; - -} diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/events.h b/ydb/core/tx/columnshard/engines/writer/buffer/events.h index d880b859aaa2..432501a47555 100644 --- a/ydb/core/tx/columnshard/engines/writer/buffer/events.h +++ b/ydb/core/tx/columnshard/engines/writer/buffer/events.h @@ -9,34 +9,6 @@ #include #include -namespace NKikimr::NColumnShard::NWriting { - -class TEvAddInsertedDataToBuffer - : public NActors::TEventLocal { -private: - YDB_READONLY_DEF(std::shared_ptr, WriteData); - YDB_READONLY_DEF(std::shared_ptr, RecordBatch); - YDB_ACCESSOR_DEF(std::vector, BlobsToWrite); - -public: - explicit TEvAddInsertedDataToBuffer(const std::shared_ptr& writeData, std::vector&& blobs, - const std::shared_ptr& recordBatch) - : WriteData(writeData) - , RecordBatch(recordBatch) - , BlobsToWrite(blobs) { - } -}; - -class TEvFlushBuffer: public NActors::TEventLocal { -private: - static inline NActors::NTests::TGlobalScheduledEvents::TRegistrator TestScheduledEventRegistrator = - (ui32)NColumnShard::TEvPrivate::EEv::EvWritingFlushBuffer; - -public: -}; - -} // namespace NKikimr::NColumnShard::NWriting - namespace NKikimr::NOlap::NWritingPortions { class TEvAddInsertedDataToBuffer diff --git a/ydb/core/tx/columnshard/engines/writer/buffer/ya.make b/ydb/core/tx/columnshard/engines/writer/buffer/ya.make index e346e6e417ea..80b67610dc91 100644 --- a/ydb/core/tx/columnshard/engines/writer/buffer/ya.make +++ b/ydb/core/tx/columnshard/engines/writer/buffer/ya.make @@ -1,7 +1,6 @@ LIBRARY() SRCS( - actor.cpp actor2.cpp events.cpp ) diff --git a/ydb/core/tx/columnshard/operations/common/context.h b/ydb/core/tx/columnshard/operations/common/context.h index f32447c959e4..de7308ec148e 100644 --- a/ydb/core/tx/columnshard/operations/common/context.h +++ b/ydb/core/tx/columnshard/operations/common/context.h @@ -9,7 +9,6 @@ namespace NKikimr::NOlap { class TWritingContext { private: YDB_READONLY(ui64, TabletId, 0); - YDB_READONLY(NActors::TActorId, BufferizationInsertionActorId, NActors::TActorId()); YDB_READONLY(NActors::TActorId, BufferizationPortionsActorId, NActors::TActorId()); YDB_READONLY(NActors::TActorId, TabletActorId, NActors::TActorId()); YDB_READONLY_DEF(std::shared_ptr, ActualSchema); @@ -37,10 +36,8 @@ class TWritingContext { TWritingContext(const ui64 tabletId, const NActors::TActorId& tabletActorId, const std::shared_ptr& actualSchema, const std::shared_ptr& operators, const std::shared_ptr& splitterCounters, const std::shared_ptr& writingCounters, const TSnapshot& applyToSnapshot, - const std::shared_ptr& activityChecker, const bool noTxWrite, - const NActors::TActorId& bufferizationInsertionActorId, const NActors::TActorId& bufferizationPortionsActorId) + const std::shared_ptr& activityChecker, const bool noTxWrite, const NActors::TActorId& bufferizationPortionsActorId) : TabletId(tabletId) - , BufferizationInsertionActorId(bufferizationInsertionActorId) , BufferizationPortionsActorId(bufferizationPortionsActorId) , TabletActorId(tabletActorId) , ActualSchema(actualSchema) diff --git a/ydb/core/tx/columnshard/operations/events.cpp b/ydb/core/tx/columnshard/operations/events.cpp index ed662c1fc381..d94c51709366 100644 --- a/ydb/core/tx/columnshard/operations/events.cpp +++ b/ydb/core/tx/columnshard/operations/events.cpp @@ -24,7 +24,6 @@ TWriteResult::TWriteResult(const std::shared_ptr& writeMet , PKBatch(pkBatch) , RecordsCount(recordsCount) { AFL_VERIFY(WriteMeta); - WriteMeta->OnStage(NEvWrite::EWriteStage::Result); } } // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/operations/events.h b/ydb/core/tx/columnshard/operations/events.h index d91f6e84772f..e32645ce1490 100644 --- a/ydb/core/tx/columnshard/operations/events.h +++ b/ydb/core/tx/columnshard/operations/events.h @@ -97,7 +97,6 @@ class TInsertedPortions { AFL_VERIFY(WriteResults.size()); std::optional pathId; for (auto&& i : WriteResults) { - i.GetWriteMeta().OnStage(NEvWrite::EWriteStage::Finished); AFL_VERIFY(!i.GetWriteMeta().HasLongTxId()); if (!pathId) { pathId = i.GetWriteMeta().GetPathId().InternalPathId; diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp index f7b4502d5b2c..12af80633ab3 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.cpp @@ -36,6 +36,7 @@ std::optional> TBuildSlicesTask::BuildSlic void TBuildSlicesTask::ReplyError(const TString& message, const NColumnShard::TEvPrivate::TEvWriteBlobsResult::EErrorClass errorClass) { AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "error_on_TBuildSlicesTask")("message", message)("class", (ui32)errorClass); + WriteData.GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::SlicesError); auto writeDataPtr = std::make_shared(std::move(WriteData)); TWritingBuffer buffer(writeDataPtr->GetBlobsAction(), { std::make_shared(*writeDataPtr) }); auto result = @@ -116,6 +117,7 @@ void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { if (OriginalBatch->num_rows() == 0) { NColumnShard::TWriteResult wResult(WriteData.GetWriteMetaPtr(), WriteData.GetSize(), nullptr, true, 0); NColumnShard::TInsertedPortions pack({ wResult }, {}); + WriteData.GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::SlicesReady); auto result = std::make_unique( NKikimrProto::EReplyStatus::OK, nullptr, std::move(pack)); NActors::TActivationContext::AsActorContext().Send(Context.GetTabletActorId(), result.release()); @@ -140,6 +142,7 @@ void TBuildSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) { } auto writeController = std::make_shared( Context.GetTabletActorId(), WriteData.GetBlobsAction(), wResult, std::move(portions)); + WriteData.GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::SlicesReady); if (WriteData.GetBlobsAction()->NeedDraftTransaction()) { TActorContext::AsActorContext().Send( Context.GetTabletActorId(), std::make_unique(writeController)); diff --git a/ydb/core/tx/columnshard/operations/slice_builder/builder.h b/ydb/core/tx/columnshard/operations/slice_builder/builder.h index 5d6bb4159777..7c3b6d4162db 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/builder.h +++ b/ydb/core/tx/columnshard/operations/slice_builder/builder.h @@ -31,7 +31,7 @@ class TBuildSlicesTask: public NConveyor::ITask, public NColumnShard::TMonitorin , TabletId(context.GetTabletId()) , OriginalBatch(batch) , Context(context) { - WriteData.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::BuildSlices); + WriteData.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::SlicesConstruction); } }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp index 1661a6bf831c..00983e46899e 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp +++ b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.cpp @@ -189,6 +189,7 @@ void TBuildPackSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) for (auto&& unit : WriteUnits) { const auto& originalBatch = unit.GetBatch(); if (originalBatch->num_rows() == 0) { + unit.GetData()->GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::PackSlicesReady); writeResults.emplace_back(unit.GetData()->GetWriteMetaPtr(), unit.GetData()->GetSize(), nullptr, true, 0); continue; } @@ -221,6 +222,10 @@ void TBuildPackSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) } } if (!cancelWritingReason) { + for (auto&& unit : WriteUnits) { + unit.GetData()->GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::PackSlicesReady); + } + auto actions = WriteUnits.front().GetData()->GetBlobsAction(); auto writeController = std::make_shared(Context.GetTabletActorId(), actions, std::move(writeResults), std::move(portionsToWrite)); @@ -231,6 +236,9 @@ void TBuildPackSlicesTask::DoExecute(const std::shared_ptr& /*taskPtr*/) TActorContext::AsActorContext().Register(NColumnShard::CreateWriteActor(TabletId, writeController, TInstant::Max())); } } else { + for (auto&& unit : WriteUnits) { + unit.GetData()->GetWriteMetaPtr()->OnStage(NEvWrite::EWriteStage::PackSlicesError); + } for (auto&& i : writeResults) { i.SetErrorMessage(cancelWritingReason, false); } diff --git a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h index a511f92395ff..48e53cb503cb 100644 --- a/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h +++ b/ydb/core/tx/columnshard/operations/slice_builder/pack_builder.h @@ -50,7 +50,7 @@ class TBuildPackSlicesTask: public NConveyor::ITask, public NColumnShard::TMonit , Context(context) { AFL_VERIFY(WriteUnits.size()); for (auto&& i : WriteUnits) { - i.GetData()->MutableWriteMeta().OnStage(NEvWrite::EWriteStage::BuildSlicesPack); + i.GetData()->MutableWriteMeta().OnStage(NEvWrite::EWriteStage::PackSlicesConstruction); } } }; diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index 1f7d3a6c3f9b..fa9bc3255c6a 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -19,8 +19,7 @@ bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) co writeOperation->SetBehaviour(Behaviour); NOlap::TWritingContext wContext(owner->TabletID(), owner->SelfId(), Schema, owner->StoragesManager, owner->Counters.GetIndexationCounters().SplitterCounters, owner->Counters.GetCSCounters().WritingCounters, NOlap::TSnapshot::Max(), - writeOperation->GetActivityChecker(), Behaviour == EOperationBehaviour::NoTxWrite, owner->BufferizationInsertionWriteActorId, - owner->BufferizationPortionsWriteActorId); + writeOperation->GetActivityChecker(), Behaviour == EOperationBehaviour::NoTxWrite, owner->BufferizationPortionsWriteActorId); ArrowData->SetSeparationPoints(owner->GetIndexAs().GetGranulePtrVerified(PathId.InternalPathId)->GetBucketPositions()); writeOperation->Start(*owner, ArrowData, SourceId, wContext); return true; diff --git a/ydb/core/tx/data_events/common/signals_flow.cpp b/ydb/core/tx/data_events/common/signals_flow.cpp index d043c9c4a98c..a3c4149e24bc 100644 --- a/ydb/core/tx/data_events/common/signals_flow.cpp +++ b/ydb/core/tx/data_events/common/signals_flow.cpp @@ -5,22 +5,10 @@ namespace NKikimr::NEvWrite { TWriteFlowCounters::TWriteFlowCounters() - : TBase("CSWriteFlow") { + : TBase("CSWriteFlow") + , Tracing(*this, "write_state") { DurationToAbort = TBase::GetDeriviative("Aborted/SumDuration"); DurationToFinish = TBase::GetDeriviative("Finished/SumDuration"); - for (auto&& i : GetEnumAllValues()) { - auto sub = CreateSubGroup("stage", ::ToString(i)); - CountByWriteStage.emplace_back(sub.GetValue("Count")); - WriteStageAdd.emplace_back(sub.GetDeriviative("Moving/Count")); - DurationToStage.emplace_back(sub.GetHistogram("DurationToStageMs", NMonitoring::ExponentialHistogram(18, 2, 1))); - CountByStageMoving.emplace_back(); - CountByStageDuration.emplace_back(); - for (auto&& to : GetEnumAllValues()) { - auto subTo = sub.CreateSubGroup("stage_to", ::ToString(to)); - CountByStageMoving.back().emplace_back(subTo.GetDeriviative("Transfers/Count")); - CountByStageDuration.back().emplace_back(subTo.GetDeriviative("Transfers/Duration/Ms")); - } - } } } // namespace NKikimr::NEvWrite diff --git a/ydb/core/tx/data_events/common/signals_flow.h b/ydb/core/tx/data_events/common/signals_flow.h index fcf6b9426819..2d54b2fa2d19 100644 --- a/ydb/core/tx/data_events/common/signals_flow.h +++ b/ydb/core/tx/data_events/common/signals_flow.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace NKikimr::NEvWrite { @@ -9,39 +10,30 @@ enum class EWriteStage { Started, BuildBatch, WaitFlush, - BuildSlices, - BuildSlicesPack, - Result, - Finished, + SlicesConstruction, + SlicesReady, + SlicesError, + PackSlicesConstruction, + PackSlicesReady, + PackSlicesError, + SuccessWritingToLocalDB, + FailWritingToLocalDB, + Replied, Aborted, - Replied + Finished }; class TWriteFlowCounters: public NColumnShard::TCommonCountersOwner { private: using TBase = TCommonCountersOwner; - std::vector CountByWriteStage; - std::vector WriteStageAdd; - std::vector> CountByStageMoving; - std::vector> CountByStageDuration; - std::vector DurationToStage; NMonitoring::TDynamicCounters::TCounterPtr DurationToFinish; NMonitoring::TDynamicCounters::TCounterPtr DurationToAbort; + NOlap::NCounters::TStateSignalsOperator Tracing; public: - void OnStageMove(const EWriteStage fromStage, const EWriteStage toStage, const TDuration d) const { - CountByWriteStage[(ui32)fromStage]->Sub(1); - CountByWriteStage[(ui32)toStage]->Add(1); - WriteStageAdd[(ui32)toStage]->Add(1); - DurationToStage[(ui32)toStage]->Collect(d.MilliSeconds()); - CountByStageMoving[(ui32)fromStage][(ui32)toStage]->Add(1); - CountByStageDuration[(ui32)fromStage][(ui32)toStage]->Add(d.MilliSeconds()); - } - - void OnWritingStart(const EWriteStage stage) const { - WriteStageAdd[(ui32)stage]->Add(1); - CountByWriteStage[(ui32)stage]->Add(1); + NOlap::NCounters::TStateSignalsOperator& MutableTracing() { + return Tracing; } void OnWriteFinished(const TDuration d) const { diff --git a/ydb/core/tx/data_events/write_data.cpp b/ydb/core/tx/data_events/write_data.cpp index 919f168bc782..9d5acdade5c1 100644 --- a/ydb/core/tx/data_events/write_data.cpp +++ b/ydb/core/tx/data_events/write_data.cpp @@ -20,18 +20,11 @@ TWriteData::TWriteData(const std::shared_ptr& writeMeta, IDataContai } void TWriteMeta::OnStage(const EWriteStage stage) const { - if (stage == CurrentStage) { - return; - } - AFL_VERIFY((ui32)stage > (ui32)CurrentStage)("from", CurrentStage)("to", stage); - const TMonotonic nextStageInstant = TMonotonic::Now(); - Counters->OnStageMove(CurrentStage, stage, nextStageInstant - LastStageInstant); - CurrentStage = stage; - LastStageInstant = nextStageInstant; + StateGuard.SetState(stage); if (stage == EWriteStage::Finished) { - Counters->OnWriteFinished(nextStageInstant - WriteStartInstant); + Counters->OnWriteFinished(TMonotonic::Now() - WriteStartInstant); } else if (stage == EWriteStage::Aborted) { - Counters->OnWriteAborted(nextStageInstant - WriteStartInstant); + Counters->OnWriteAborted(TMonotonic::Now() - WriteStartInstant); } } diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index a3c58ca85242..332f395e31c4 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -4,14 +4,14 @@ #include #include -#include +#include #include #include #include #include #include -#include +#include #include @@ -51,14 +51,13 @@ class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TN YDB_READONLY(TMonotonic, WriteStartInstant, TMonotonic::Now()); std::optional LockId; const std::shared_ptr Counters; - mutable TMonotonic LastStageInstant = TMonotonic::Now(); - mutable EWriteStage CurrentStage = EWriteStage::Created; + mutable NOlap::NCounters::TStateSignalsOperator::TGuard StateGuard; public: void OnStage(const EWriteStage stage) const; ~TWriteMeta() { - if (CurrentStage != EWriteStage::Replied) { + if (StateGuard.GetStage() != EWriteStage::Finished) { OnStage(EWriteStage::Aborted); } } @@ -89,15 +88,15 @@ class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TN } } - TWriteMeta(const ui64 writeId, const NColumnShard::TUnifiedPathId& pathId, const NActors::TActorId& source, const std::optional granuleShardingVersion, - const TString& writingIdentifier, const std::shared_ptr& counters) + TWriteMeta(const ui64 writeId, const NColumnShard::TUnifiedPathId& pathId, const NActors::TActorId& source, + const std::optional granuleShardingVersion, const TString& writingIdentifier, const std::shared_ptr& counters) : WriteId(writeId) , PathId(pathId) , Source(source) , GranuleShardingVersion(granuleShardingVersion) , Id(writingIdentifier) - , Counters(counters) { - Counters->OnWritingStart(CurrentStage); + , Counters(counters) + , StateGuard(Counters->MutableTracing().BuildGuard(NEvWrite::EWriteStage::Created)) { } }; diff --git a/ydb/core/tx/general_cache/service/counters.h b/ydb/core/tx/general_cache/service/counters.h index 12c012e62c80..8a1e71188451 100644 --- a/ydb/core/tx/general_cache/service/counters.h +++ b/ydb/core/tx/general_cache/service/counters.h @@ -1,4 +1,7 @@ #pragma once +#include + +#include #include namespace NKikimr::NGeneralCache::NPrivate { @@ -6,9 +9,28 @@ namespace NKikimr::NGeneralCache::NPrivate { class TManagerCounters: public NColumnShard::TCommonCountersOwner { private: using TBase = NColumnShard::TCommonCountersOwner; + const NPublic::TConfig Config; const NMonitoring::THistogramPtr RequestDuration; + const std::shared_ptr TotalInFlight = std::make_shared(); + const std::shared_ptr QueueObjectsCount = std::make_shared(); public: + bool CheckTotalLimit() const { + return GetTotalInFlight()->Val() < Config.GetDirectInflightGlobalLimit(); + } + + const std::shared_ptr& GetTotalInFlight() const { + return TotalInFlight; + } + + const std::shared_ptr& GetQueueObjectsCount() const { + return QueueObjectsCount; + } + + const NPublic::TConfig& GetConfig() const { + return Config; + } + void OnRequestFinished(const TDuration d) const { RequestDuration->Collect(d.MicroSeconds()); } @@ -25,6 +47,7 @@ class TManagerCounters: public NColumnShard::TCommonCountersOwner { const NMonitoring::TDynamicCounters::TCounterPtr ObjectsQueueSize; const NMonitoring::TDynamicCounters::TCounterPtr IncomingRequestsCount; const NMonitoring::TDynamicCounters::TCounterPtr IncomingAbortedRequestsCount; + const NMonitoring::TDynamicCounters::TCounterPtr DirectObjects; const NMonitoring::TDynamicCounters::TCounterPtr DirectRequests; const NMonitoring::TDynamicCounters::TCounterPtr AbortedRequests; const NMonitoring::TDynamicCounters::TCounterPtr AdditionalObjectInfo; @@ -33,8 +56,9 @@ class TManagerCounters: public NColumnShard::TCommonCountersOwner { const NMonitoring::TDynamicCounters::TCounterPtr NoExistsObject; const NMonitoring::TDynamicCounters::TCounterPtr FailedObject; - TManagerCounters(NColumnShard::TCommonCountersOwner& base) + TManagerCounters(NColumnShard::TCommonCountersOwner& base, const NPublic::TConfig& config) : TBase(base, "signals_owner", "manager") + , Config(config) , RequestDuration(TBase::GetHistogram("Requests/Duration/Us", NMonitoring::ExponentialHistogram(15, 2, 16))) , RequestCacheMiss(TBase::GetDeriviative("Cache/Request/Miss/Count")) , RequestCacheHit(TBase::GetDeriviative("Cache/Request/Hit/Count")) @@ -48,7 +72,8 @@ class TManagerCounters: public NColumnShard::TCommonCountersOwner { , ObjectsQueueSize(TBase::GetValue("ObjectsQueue/Size/Count")) , IncomingRequestsCount(TBase::GetDeriviative("Incoming/Requests/Count")) , IncomingAbortedRequestsCount(TBase::GetDeriviative("Incoming/AbortedRequests/Count")) - , DirectRequests(TBase::GetDeriviative("DirectRequest/Count")) + , DirectObjects(TBase::GetDeriviative("Direct/Object/Count")) + , DirectRequests(TBase::GetDeriviative("Direct/Request/Count")) , AbortedRequests(TBase::GetDeriviative("AbortedRequest/Count")) , AdditionalObjectInfo(TBase::GetDeriviative("AdditionalInfo/Count")) , RemovedObjectInfo(TBase::GetDeriviative("RemovedInfo/Count")) @@ -64,10 +89,10 @@ class TActorCounters: public NColumnShard::TCommonCountersOwner { YDB_READONLY_DEF(std::shared_ptr, Manager); public: - TActorCounters(const TString& cacheName, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& baseCounters) + TActorCounters(const TString& cacheName, const TIntrusivePtr<::NMonitoring::TDynamicCounters>& baseCounters, const NPublic::TConfig& config) : TBase("general_cache", baseCounters) { DeepSubGroup("cache_name", cacheName); - Manager = std::make_shared(*this); + Manager = std::make_shared(*this, config); } }; diff --git a/ydb/core/tx/general_cache/service/manager.h b/ydb/core/tx/general_cache/service/manager.h index 07f405e83c4c..8eafcf4eb06d 100644 --- a/ydb/core/tx/general_cache/service/manager.h +++ b/ydb/core/tx/general_cache/service/manager.h @@ -19,18 +19,52 @@ class TRequest: public NColumnShard::TMonitoringObjectsCounter using TObject = typename TPolicy::TObject; using EConsumer = typename TPolicy::EConsumer; using ICallback = NPublic::ICallback; + using TSourceId = typename TPolicy::TSourceId; static inline TAtomicCounter Counter = 0; YDB_READONLY(ui64, RequestId, Counter.Inc()); YDB_READONLY(TMonotonic, Created, TMonotonic::Now()); - YDB_READONLY_DEF(THashSet, Wait); + THashSet Cookies; + THashMap> Wait; THashMap Result; THashSet Removed; THashMap Errors; + TPositiveControlInteger WaitObjectsCount; + std::shared_ptr Callback; const EConsumer Consumer; + bool RemoveAddrOnFinished(const TAddress addr) { + auto itSource = Wait.find(TPolicy::GetSourceId(addr)); + AFL_VERIFY(itSource != Wait.end()); + AFL_VERIFY(itSource->second.erase(addr)); + if (itSource->second.empty()) { + Wait.erase(itSource); + } + if (Wait.empty()) { + AFL_VERIFY(WaitObjectsCount.Val() == 0); + Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); + return true; + } else { + return false; + } + } + public: + ui64 GetWaitObjectsCount() { + return WaitObjectsCount.Val(); + } + + const THashMap>& GetWaitBySource() const { + return Wait; + } + + const THashSet& GetWaitBySource(const TSourceId sourceId) const { + auto it = Wait.find(sourceId); + AFL_VERIFY(it != Wait.end()); + return it->second; + } + bool IsAborted() const { return Callback->IsAborted(); } @@ -40,138 +74,265 @@ class TRequest: public NColumnShard::TMonitoringObjectsCounter } [[nodiscard]] bool AddResult(const TAddress& addr, const TObject& obj) { - AFL_VERIFY(Wait.erase(addr)); AFL_VERIFY(Result.emplace(addr, obj).second); - if (Wait.empty()) { - Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); - } - return Wait.empty(); + WaitObjectsCount.Dec(); + return RemoveAddrOnFinished(addr); } [[nodiscard]] bool AddRemoved(const TAddress& addr) { - AFL_VERIFY(Wait.erase(addr)); AFL_VERIFY(Removed.emplace(addr).second); - if (Wait.empty()) { - Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); - } - return Wait.empty(); + WaitObjectsCount.Dec(); + return RemoveAddrOnFinished(addr); } [[nodiscard]] bool AddError(const TAddress& addr, const TString& errorMessage) { - AFL_VERIFY(Wait.erase(addr)); AFL_VERIFY(Errors.emplace(addr, errorMessage).second); - if (Wait.empty()) { - Callback->OnResultReady(std::move(Result), std::move(Removed), std::move(Errors)); - } - return Wait.empty(); + WaitObjectsCount.Dec(); + return RemoveAddrOnFinished(addr); } TRequest(THashSet&& addresses, std::shared_ptr&& callback, const EConsumer consumer) - : Wait(addresses) - , Callback(callback) + : Callback(callback) , Consumer(consumer) { + for (auto&& i : addresses) { + Wait[TPolicy::GetSourceId(i)].emplace(i); + WaitObjectsCount.Inc(); + } } }; template -class TManager { +class TSourceInfo { private: using TAddress = typename TPolicy::TAddress; using TObject = typename TPolicy::TObject; + using TSourceId = typename TPolicy::TSourceId; using EConsumer = typename TPolicy::EConsumer; using TRequest = TRequest; - const NPublic::TConfig Config; - const TString CacheName = TPolicy::GetCacheName(); - const std::shared_ptr Counters; - std::shared_ptr> ObjectsProcessor; - TLRUCache Cache; + const TSourceId SourceId; + static inline TAtomicCounter Counter = 0; + const ui64 Cookie = Counter.Inc(); THashMap>> RequestedObjects; - THashSet RequestsInProgress; - std::deque> RequestsQueue; - TPositiveControlInteger QueueObjectsCount; + YDB_READONLY_DEF(std::deque>, RequestsQueue); + YDB_READONLY_DEF(THashSet, RequestsInProgress); + THashMap>> RequestsByCookies; + const std::shared_ptr Counters; + const std::shared_ptr> ObjectsProcessor; + +public: + ui64 GetCookie() const { + return Cookie; + } + + TSourceInfo(const std::shared_ptr& counters, const TSourceId& sourceId, + const std::shared_ptr>& objectsProcessor) + : SourceId(sourceId) + , Counters(counters) + , ObjectsProcessor(objectsProcessor) { + } + + void EnqueueRequest(const std::shared_ptr& request) { + Counters->GetQueueObjectsCount()->Add(request->GetWaitBySource(SourceId).size()); + RequestsQueue.emplace_back(request); + } + + void AddObjects(THashMap&& add, const bool isAdditional, const TMonotonic now) { + for (auto&& i : add) { + auto it = RequestedObjects.find(i.first); + if (it == RequestedObjects.end()) { + continue; + } + for (auto&& r : it->second) { + if (!r->AddResult(i.first, i.second)) { + continue; + } + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + if (isAdditional) { + Counters->AdditionalObjectInfo->Inc(); + } else { + Counters->FetchedObject->Inc(); + } + } + RequestedObjects.erase(it); + Counters->GetTotalInFlight()->Dec(); + } + } + + void FailObjects(THashMap&& failed, const TMonotonic now) { + for (auto&& i : failed) { + auto it = RequestedObjects.find(i.first); + AFL_VERIFY(it != RequestedObjects.end()); + for (auto&& r : it->second) { + Counters->FailedObject->Inc(); + if (r->AddError(i.first, i.second)) { + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + } + } + RequestedObjects.erase(it); + Counters->GetTotalInFlight()->Dec(); + } + } + + void RemoveObjects(THashSet&& remove, const bool isAdditional, const TMonotonic now) { + for (auto&& i : remove) { + auto it = RequestedObjects.find(i); + if (it == RequestedObjects.end()) { + continue; + } + for (auto&& r : it->second) { + if (!r->AddRemoved(i)) { + continue; + } + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + if (isAdditional) { + Counters->RemovedObjectInfo->Inc(); + } else { + Counters->NoExistsObject->Inc(); + } + } + RequestedObjects.erase(it); + Counters->GetTotalInFlight()->Dec(); + } + } + + void Abort() { + const TMonotonic now = TMonotonic::Now(); + for (auto&& i : RequestsQueue) { + for (auto&& objAddr : i->GetWaitBySource(SourceId)) { + Y_UNUSED(i->AddError(objAddr, "source broken: " + ::ToString(SourceId))); + } + } + RequestsQueue.clear(); + for (auto&& [objAddr, requests] : RequestedObjects) { + for (auto&& r : requests) { + Counters->FailedObject->Inc(); + if (r->AddError(objAddr, "source broken: " + ::ToString(SourceId))) { + RequestsInProgress.erase(r->GetRequestId()); + Counters->OnRequestFinished(now - r->GetCreated()); + } + } + } + AFL_VERIFY(RequestsInProgress.empty()); + Counters->GetTotalInFlight()->Sub(RequestedObjects.size()); + RequestedObjects.clear(); + } + + ~TSourceInfo() { + AFL_VERIFY(RequestedObjects.empty()); + } void DrainQueue() { THashMap> requestedAddresses; - while (RequestsQueue.size() && RequestedObjects.size() < Config.GetDirectInflightLimit()) { + while (RequestsQueue.size() && RequestedObjects.size() < Counters->GetConfig().GetDirectInflightSourceLimit() && + Counters->CheckTotalLimit()) { auto request = std::move(RequestsQueue.front()); RequestsQueue.pop_front(); - QueueObjectsCount.Sub(request->GetWait().size()); + auto& sourceWaitObjects = request->GetWaitBySource(SourceId); + Counters->GetQueueObjectsCount()->Sub(sourceWaitObjects.size()); if (request->IsAborted()) { Counters->AbortedRequests->Inc(); continue; } AFL_VERIFY(RequestsInProgress.emplace(request->GetRequestId()).second); auto& addresses = requestedAddresses[request->GetConsumer()]; - for (auto&& i : request->GetWait()) { + Counters->DirectRequests->Inc(); + for (auto&& i : sourceWaitObjects) { auto it = RequestedObjects.find(i); if (it == RequestedObjects.end()) { it = RequestedObjects.emplace(i, std::vector>()).first; + Counters->GetTotalInFlight()->Inc(); AFL_VERIFY(addresses.emplace(i).second); + Counters->DirectObjects->Inc(); } it->second.emplace_back(request); } } - ObjectsProcessor->AskData(std::move(requestedAddresses), ObjectsProcessor); - Counters->DirectRequests->Inc(); - Counters->RequestsQueueSize->Set(RequestsQueue.size()); - Counters->ObjectsQueueSize->Set(QueueObjectsCount.Val()); - Counters->ObjectsInFlight->Set(RequestedObjects.size()); - Counters->RequestsInFlight->Set(RequestsInProgress.size()); - Counters->CacheSizeCount->Set(Cache.Size()); - Counters->CacheSizeBytes->Set(Cache.TotalSize()); + ObjectsProcessor->AskData(std::move(requestedAddresses), ObjectsProcessor, Cookie); + Counters->ObjectsQueueSize->Set(Counters->GetQueueObjectsCount()->Val()); + Counters->ObjectsInFlight->Set(Counters->GetTotalInFlight()->Val()); } +}; - void AddObjects(THashMap&& add, const bool isAdditional, const TMonotonic now) { - for (auto&& i : add) { - auto it = RequestedObjects.find(i.first); - Cache.Insert(i.first, i.second); - if (it != RequestedObjects.end()) { - for (auto&& r : it->second) { - if (r->AddResult(i.first, i.second)) { - RequestsInProgress.erase(r->GetRequestId()); - Counters->OnRequestFinished(now - r->GetCreated()); - if (isAdditional) { - Counters->AdditionalObjectInfo->Inc(); - } else { - Counters->FetchedObject->Inc(); - } - } +template +class TManager { +private: + using TSourceId = typename TPolicy::TSourceId; + using TAddress = typename TPolicy::TAddress; + using TObject = typename TPolicy::TObject; + using EConsumer = typename TPolicy::EConsumer; + using TRequest = TRequest; + using TSourceInfo = TSourceInfo; - } - RequestedObjects.erase(it); + const TString CacheName = TPolicy::GetCacheName(); + const std::shared_ptr Counters; + std::shared_ptr> ObjectsProcessor; + TLRUCache Cache; + + THashMap SourcesInfo; + + void DrainQueue(const TSourceId sourceId) { + MutableSourceInfo(sourceId).DrainQueue(); + } + + void DrainQueue() { + for (auto&& i : SourcesInfo) { + if (!Counters->CheckTotalLimit()) { + return; } + i.second.DrainQueue(); } } - void RemoveObjects(THashSet&& remove, const bool isAdditional, const TMonotonic now) { - for (auto&& i : remove) { - auto it = RequestedObjects.find(i); - if (it != RequestedObjects.end()) { - for (auto&& r : it->second) { - if (r->AddRemoved(i)) { - RequestsInProgress.erase(r->GetRequestId()); - Counters->OnRequestFinished(now - r->GetCreated()); - if (isAdditional) { - Counters->RemovedObjectInfo->Inc(); - } else { - Counters->NoExistsObject->Inc(); - } - } - } - RequestedObjects.erase(it); - } + TSourceInfo& MutableSourceInfo(const TSourceId sourceId) { + auto it = SourcesInfo.find(sourceId); + AFL_VERIFY(it != SourcesInfo.end()); + return it->second; + } + + void AddObjects(const TSourceId sourceId, THashMap&& add, const bool isAdditional, const TMonotonic now) { + for (auto&& i : add) { + Cache.Insert(i.first, i.second); + } + Counters->CacheSizeCount->Set(Cache.Size()); + Counters->CacheSizeBytes->Set(Cache.TotalSize()); + if (isAdditional && !SourcesInfo.contains(sourceId)) { + SourcesInfo.emplace(sourceId, TSourceInfo(Counters, sourceId, ObjectsProcessor)); } + MutableSourceInfo(sourceId).AddObjects(std::move(add), isAdditional, now); + } + + void RemoveObjects(const TSourceId sourceId, THashSet&& remove, const bool isAdditional, const TMonotonic now) { + MutableSourceInfo(sourceId).RemoveObjects(std::move(remove), isAdditional, now); } public: - TManager(const NPublic::TConfig& config, const NActors::TActorId& ownerActorId, - const std::shared_ptr& counters) - : Config(config) - , Counters(counters) + TManager(const NActors::TActorId& ownerActorId, const std::shared_ptr& counters) + : Counters(counters) , ObjectsProcessor(TPolicy::BuildObjectsProcessor(ownerActorId)) - , Cache(Config.GetMemoryLimit()) { - AFL_NOTICE(NKikimrServices::GENERAL_CACHE)("event", "general_cache_manager")("owner_actor_id", ownerActorId)("config", config.DebugString()); + , Cache(Counters->GetConfig().GetMemoryLimit()) { + AFL_NOTICE(NKikimrServices::GENERAL_CACHE)("event", "general_cache_manager")("owner_actor_id", ownerActorId)( + "config", Counters->GetConfig().DebugString()); + } + + TSourceId GetSourceByCookie(const ui64 cookie) const { + for (auto&& i : SourcesInfo) { + if (i.second.GetCookie() == cookie) { + return i.first; + } + } + AFL_VERIFY(false); + return TSourceId(); + } + + void AbortSource(const TSourceId sourceId) { + if (SourcesInfo.contains(sourceId)) { + MutableSourceInfo(sourceId).Abort(); + SourcesInfo.erase(sourceId); + } } void AddRequest(const std::shared_ptr& request) { @@ -183,56 +344,64 @@ class TManager { } else { Counters->IncomingRequestsCount->Inc(); } - for (auto&& i : request->GetWait()) { - auto it = Cache.Find(i); - if (it == Cache.End()) { - Counters->ObjectCacheMiss->Inc(); - } else { - Counters->ObjectCacheHit->Inc(); - AFL_VERIFY(objectsResult.emplace(i, it.Value()).second); + for (auto&& [sourceId, addresses] : request->GetWaitBySource()) { + for (auto&& addr : addresses) { + auto it = Cache.Find(addr); + if (it == Cache.End()) { + Counters->ObjectCacheMiss->Inc(); + } else { + Counters->ObjectCacheHit->Inc(); + AFL_VERIFY(objectsResult.emplace(addr, it.Value()).second); + } } } for (auto&& i : objectsResult) { Y_UNUSED(request->AddResult(i.first, std::move(i.second))); } - if (request->GetWait().empty()) { + if (request->GetWaitBySource().empty()) { Counters->RequestCacheHit->Inc(); return; } else { Counters->RequestCacheMiss->Inc(); } - QueueObjectsCount.Add(request->GetWait().size()); - RequestsQueue.emplace_back(request); - DrainQueue(); + for (auto&& i : request->GetWaitBySource()) { + auto it = SourcesInfo.find(i.first); + if (it == SourcesInfo.end()) { + it = SourcesInfo.emplace(i.first, TSourceInfo(Counters, i.first, ObjectsProcessor)).first; + } + it->second.EnqueueRequest(request); + it->second.DrainQueue(); + } } - void OnAdditionalObjectsInfo(THashMap&& add, THashSet&& remove) { + void OnAdditionalObjectsInfo(const TSourceId sourceId, THashMap&& add, THashSet&& remove) { AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "objects_info"); const TMonotonic now = TMonotonic::Now(); - AddObjects(std::move(add), true, now); - RemoveObjects(std::move(remove), true, now); - DrainQueue(); + const bool inFlightLimitBrokenBefore = !Counters->CheckTotalLimit(); + AddObjects(sourceId, std::move(add), true, now); + RemoveObjects(sourceId, std::move(remove), true, now); + const bool inFlightLimitBrokenAfter = !Counters->CheckTotalLimit(); + if (inFlightLimitBrokenBefore && !inFlightLimitBrokenAfter) { + DrainQueue(); + } else { + DrainQueue(sourceId); + } } - void OnRequestResult(THashMap&& objects, THashSet&& removed, THashMap&& failed) { + void OnRequestResult( + const TSourceId sourceId, THashMap&& objects, THashSet&& removed, THashMap&& failed) { AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "on_result"); const TMonotonic now = TMonotonic::Now(); - AddObjects(std::move(objects), false, now); - RemoveObjects(std::move(removed), false, now); - - for (auto&& i : failed) { - auto it = RequestedObjects.find(i.first); - AFL_VERIFY(it != RequestedObjects.end()); - for (auto&& r : it->second) { - if (r->AddError(i.first, i.second)) { - RequestsInProgress.erase(r->GetRequestId()); - Counters->OnRequestFinished(now - r->GetCreated()); - Counters->FailedObject->Inc(); - } - } - RequestedObjects.erase(it); + const bool inFlightLimitBrokenBefore = !Counters->CheckTotalLimit(); + AddObjects(sourceId, std::move(objects), false, now); + RemoveObjects(sourceId, std::move(removed), false, now); + MutableSourceInfo(sourceId).FailObjects(std::move(failed), now); + const bool inFlightLimitBrokenAfter = !Counters->CheckTotalLimit(); + if (inFlightLimitBrokenBefore && !inFlightLimitBrokenAfter) { + DrainQueue(); + } else { + DrainQueue(sourceId); } - DrainQueue(); } }; diff --git a/ydb/core/tx/general_cache/service/service.h b/ydb/core/tx/general_cache/service/service.h index f95f120bdeb0..77d5cd917fd8 100644 --- a/ydb/core/tx/general_cache/service/service.h +++ b/ydb/core/tx/general_cache/service/service.h @@ -27,10 +27,18 @@ class TDistributor: public TActorBootstrapped> { Manager->AddRequest(std::make_shared(ev->Get()->ExtractAddresses(), ev->Get()->ExtractCallback(), ev->Get()->GetConsumer())); } void HandleMain(NSource::TEvents::TEvObjectsInfo::TPtr& ev) { - Manager->OnRequestResult(ev->Get()->ExtractObjects(), ev->Get()->ExtractRemoved(), ev->Get()->ExtractErrors()); + Manager->OnRequestResult(ev->Get()->GetSourceId(), ev->Get()->ExtractObjects(), ev->Get()->ExtractRemoved(), ev->Get()->ExtractErrors()); } void HandleMain(NSource::TEvents::TEvAdditionalObjectsInfo::TPtr& ev) { - Manager->OnAdditionalObjectsInfo(ev->Get()->ExtractAddObjects(), ev->Get()->ExtractRemoveObjects()); + Manager->OnAdditionalObjectsInfo(ev->Get()->GetSourceId(), ev->Get()->ExtractAddObjects(), ev->Get()->ExtractRemoveObjects()); + } + + void HandleMain(NActors::TEvents::TEvUndelivered::TPtr& ev) { + Manager->AbortSource(Manager->GetSourceByCookie(ev->Cookie)); + } + + void HandleMain(NPublic::TEvents::TEvKillSource::TPtr& ev) { + Manager->AbortSource(ev->Get()->GetSourceId()); } public: @@ -39,8 +47,10 @@ class TDistributor: public TActorBootstrapped> { // ("workers", Workers.size())("waiting", Waiting.size())("actor_id", SelfId()); switch (ev->GetTypeRewrite()) { hFunc(NPublic::TEvents::TEvAskData, HandleMain); + hFunc(NPublic::TEvents::TEvKillSource, HandleMain); hFunc(NSource::TEvents::TEvObjectsInfo, HandleMain); hFunc(NSource::TEvents::TEvAdditionalObjectsInfo, HandleMain); + hFunc(NActors::TEvents::TEvUndelivered, HandleMain); default: AFL_ERROR(NKikimrServices::TX_CONVEYOR)("problem", "unexpected event for general cache")("ev_type", ev->GetTypeName()); break; @@ -49,14 +59,14 @@ class TDistributor: public TActorBootstrapped> { TDistributor(const NPublic::TConfig& config, const TIntrusivePtr<::NMonitoring::TDynamicCounters> conveyorSignals) : Config(config) - , Counters(TPolicy::GetCacheName(), conveyorSignals) { + , Counters(TPolicy::GetCacheName(), conveyorSignals, config) { } ~TDistributor() { } void Bootstrap() { - Manager = std::make_unique(Config, TBase::SelfId(), Counters.GetManager()); + Manager = std::make_unique(TBase::SelfId(), Counters.GetManager()); TBase::Become(&TDistributor::StateMain); } }; diff --git a/ydb/core/tx/general_cache/source/abstract.h b/ydb/core/tx/general_cache/source/abstract.h index 3366b073735c..c185746bfae9 100644 --- a/ydb/core/tx/general_cache/source/abstract.h +++ b/ydb/core/tx/general_cache/source/abstract.h @@ -11,24 +11,26 @@ class IObjectsProcessor { private: using TAddress = typename TPolicy::TAddress; using TObject = typename TPolicy::TObject; + using TSourceId = typename TPolicy::TSourceId; using EConsumer = typename TPolicy::EConsumer; using TSelf = IObjectsProcessor; virtual void DoAskData( - const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const = 0; - virtual void DoOnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr, const ui64 cookie) const = 0; + virtual void DoOnReceiveData(const TSourceId sourceId, THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errorAddresses) const = 0; public: virtual ~IObjectsProcessor() = default; - void AskData(const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr) const { - DoAskData(objectAddressesByConsumer, selfPtr); + void AskData(const THashMap>& objectAddressesByConsumer, const std::shared_ptr& selfPtr, + const ui64 cookie) const { + DoAskData(objectAddressesByConsumer, selfPtr, cookie); } - void OnReceiveData(THashMap&& objectAddresses, THashSet&& removedAddresses, + void OnReceiveData(const TSourceId sourceId, THashMap&& objectAddresses, THashSet&& removedAddresses, THashMap&& errorAddresses) const { - DoOnReceiveData(std::move(objectAddresses), std::move(removedAddresses), std::move(errorAddresses)); + DoOnReceiveData(sourceId, std::move(objectAddresses), std::move(removedAddresses), std::move(errorAddresses)); } }; diff --git a/ydb/core/tx/general_cache/source/events.h b/ydb/core/tx/general_cache/source/events.h index d515dbe3a163..f314de7bff77 100644 --- a/ydb/core/tx/general_cache/source/events.h +++ b/ydb/core/tx/general_cache/source/events.h @@ -13,6 +13,7 @@ template struct TEvents { using TAddress = typename TPolicy::TAddress; using TObject = typename TPolicy::TObject; + using TSourceId = typename TPolicy::TSourceId; using EConsumer = typename TPolicy::EConsumer; enum EEv { @@ -32,7 +33,13 @@ struct TEvents { bool ErrorsExtracted = false; THashMap Errors; + const TSourceId SourceId; + public: + TSourceId GetSourceId() const { + return SourceId; + } + THashMap ExtractObjects() { AFL_VERIFY(!ObjectsExtracted); ObjectsExtracted = true; @@ -51,11 +58,12 @@ struct TEvents { return std::move(Errors); } - TEvObjectsInfo(THashMap&& objects, THashSet&& removed, THashMap&& errors) + TEvObjectsInfo( + const TSourceId sourceId, THashMap&& objects, THashSet&& removed, THashMap&& errors) : Removed(std::move(removed)) , Objects(std::move(objects)) , Errors(std::move(errors)) - { + , SourceId(sourceId) { } }; @@ -67,7 +75,13 @@ struct TEvents { bool RemoveObjectsExtracted = false; THashSet RemoveObjects; + const TSourceId SourceId; + public: + TSourceId GetSourceId() const { + return SourceId; + } + THashMap ExtractAddObjects() { AFL_VERIFY(!AddObjectsExtracted); AddObjectsExtracted = true; @@ -80,10 +94,10 @@ struct TEvents { return std::move(RemoveObjects); } - TEvAdditionalObjectsInfo(THashMap&& add, THashSet&& remove) + TEvAdditionalObjectsInfo(const TSourceId sourceId, THashMap&& add, THashSet&& remove) : AddObjects(std::move(add)) , RemoveObjects(std::move(remove)) - { + , SourceId(sourceId) { } }; }; diff --git a/ydb/core/tx/general_cache/usage/config.cpp b/ydb/core/tx/general_cache/usage/config.cpp index 097f9c3ce768..d6fbf5b031c2 100644 --- a/ydb/core/tx/general_cache/usage/config.cpp +++ b/ydb/core/tx/general_cache/usage/config.cpp @@ -5,8 +5,11 @@ namespace NKikimr::NGeneralCache::NPublic { TConclusionStatus TConfig::DeserializeFromProto(const NKikimrConfig::TGeneralCacheConfig& config) { - if (config.HasDirectInflightLimit()) { - DirectInflightLimit = config.GetDirectInflightLimit(); + if (config.HasDirectInflightSourceLimit()) { + DirectInflightSourceLimit = config.GetDirectInflightSourceLimit(); + } + if (config.HasDirectInflightGlobalLimit()) { + DirectInflightGlobalLimit = config.GetDirectInflightGlobalLimit(); } if (config.HasMemoryLimit()) { MemoryLimit = config.GetMemoryLimit(); @@ -27,7 +30,8 @@ TString TConfig::DebugString() const { TStringBuilder sb; sb << "{"; sb << "MemoryLimit=" << MemoryLimit << ";"; - sb << "DirectInflightLimit=" << DirectInflightLimit << ";"; + sb << "DirectInflightSourceLimit=" << DirectInflightSourceLimit << ";"; + sb << "DirectInflightGlobalLimit=" << DirectInflightGlobalLimit << ";"; sb << "}"; return sb; } diff --git a/ydb/core/tx/general_cache/usage/config.h b/ydb/core/tx/general_cache/usage/config.h index bac2543131fb..fc239f42d7a9 100644 --- a/ydb/core/tx/general_cache/usage/config.h +++ b/ydb/core/tx/general_cache/usage/config.h @@ -9,7 +9,8 @@ namespace NKikimr::NGeneralCache::NPublic { class TConfig { private: YDB_READONLY(ui32, MemoryLimit, ((ui64)1 << 30)); - YDB_READONLY(ui32, DirectInflightLimit, 1000); + YDB_READONLY(ui32, DirectInflightSourceLimit, 2000); + YDB_READONLY(ui32, DirectInflightGlobalLimit, 20000); TConfig() = default; [[nodiscard]] TConclusionStatus DeserializeFromProto(const NKikimrConfig::TGeneralCacheConfig& config); diff --git a/ydb/core/tx/general_cache/usage/events.h b/ydb/core/tx/general_cache/usage/events.h index 4f4994c316c7..d848254ee730 100644 --- a/ydb/core/tx/general_cache/usage/events.h +++ b/ydb/core/tx/general_cache/usage/events.h @@ -14,10 +14,12 @@ template struct TEvents { using TAddress = typename TPolicy::TAddress; using EConsumer = typename TPolicy::EConsumer; + using TSourceId = typename TPolicy::TSourceId; using ICallback = ICallback; enum EEv { EvAskData = EventSpaceBegin(TKikimrEvents::ES_GENERAL_CACHE_PUBLIC), + EvKillSource, EvEnd }; @@ -51,6 +53,20 @@ struct TEvents { return std::move(Callback); } }; + + class TEvKillSource: public NActors::TEventLocal { + private: + const TSourceId SourceId; + + public: + TEvKillSource(const TSourceId sourceId) + : SourceId(sourceId) { + } + + TSourceId GetSourceId() const { + return SourceId; + } + }; }; } // namespace NKikimr::NGeneralCache::NPublic diff --git a/ydb/core/tx/general_cache/usage/service.h b/ydb/core/tx/general_cache/usage/service.h index a9ace52f1af4..b15f252e7002 100644 --- a/ydb/core/tx/general_cache/usage/service.h +++ b/ydb/core/tx/general_cache/usage/service.h @@ -11,6 +11,7 @@ class TServiceOperator { public: using TAddress = typename TPolicy::TAddress; using TObject = typename TPolicy::TObject; + using TSourceId = typename TPolicy::TSourceId; using EConsumer = typename TPolicy::EConsumer; using ICallback = NPublic::ICallback; @@ -18,16 +19,23 @@ class TServiceOperator { using TSelf = TServiceOperator; public: + static void KillSource(const TSourceId sourceId) { + AFL_VERIFY(NActors::TlsActivationContext); + auto& context = NActors::TActorContext::AsActorContext(); + context.Send(GetCurrentNodeServiceId(), new NPublic::TEvents::TEvKillSource(sourceId)); + } + static void AskObjects(const EConsumer consumer, THashSet&& addresses, std::shared_ptr&& callback) { AFL_VERIFY(NActors::TlsActivationContext); auto& context = NActors::TActorContext::AsActorContext(); context.Send(GetCurrentNodeServiceId(), new NPublic::TEvents::TEvAskData(consumer, std::move(addresses), std::move(callback))); } - static void ModifyObjects(THashMap&& add, THashSet&& remove) { + + static void ModifyObjects(const TSourceId sourceId, THashMap&& add, THashSet&& remove) { AFL_VERIFY(NActors::TlsActivationContext); auto& context = NActors::TActorContext::AsActorContext(); context.Send( - GetCurrentNodeServiceId(), new NSource::TEvents::TEvAdditionalObjectsInfo(std::move(add), std::move(remove))); + GetCurrentNodeServiceId(), new NSource::TEvents::TEvAdditionalObjectsInfo(sourceId, std::move(add), std::move(remove))); } static NActors::TActorId MakeServiceId(const ui32 nodeId) { return NActors::TActorId(nodeId, "SrvcCach" + TPolicy::GetServiceCode()); From 54274c857ca4f63f90e7c71436871bd82f86a1c4 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Mon, 30 Jun 2025 13:48:59 +0300 Subject: [PATCH 44/62] cleaning and correct backgrounds inflight control (#20344) Co-authored-by: ivanmorozov333 --- .../tx/columnshard/background_controller.cpp | 38 ++----------------- .../tx/columnshard/background_controller.h | 12 +----- .../changes/abstract/compaction_info.h | 10 +++-- .../engines/changes/compaction.cpp | 2 +- .../portion/clean_deprecated_snapshot.cpp | 2 +- 5 files changed, 14 insertions(+), 50 deletions(-) diff --git a/ydb/core/tx/columnshard/background_controller.cpp b/ydb/core/tx/columnshard/background_controller.cpp index e1c149cee399..cf972401fc51 100644 --- a/ydb/core/tx/columnshard/background_controller.cpp +++ b/ydb/core/tx/columnshard/background_controller.cpp @@ -3,8 +3,8 @@ namespace NKikimr::NColumnShard { -bool TBackgroundController::StartCompaction(const TInternalPathId pathId) { - auto [it, _] = ActiveCompactionInfo.emplace(pathId, NOlap::TPlanCompactionInfo{pathId}); +bool TBackgroundController::StartCompaction(const TInternalPathId pathId, const TString& taskId) { + auto [it, _] = ActiveCompactionInfo.emplace(pathId, NOlap::TPlanCompactionInfo{ pathId, taskId }); it->second.Start(); return true; } @@ -21,40 +21,10 @@ void TBackgroundController::FinishCompaction(const TInternalPathId pathId) { void TBackgroundController::CheckDeadlines() { for (auto&& i : ActiveCompactionInfo) { if (TMonotonic::Now() - i.second.GetStartTime() > NOlap::TCompactionLimits::CompactionTimeout) { - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction")("path_id", i.first); - Y_DEBUG_ABORT_UNLESS(false); + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_compaction")("path_id", i.first)("task_id", i.second.GetTaskId()); + AFL_VERIFY_DEBUG(false); } } } -void TBackgroundController::CheckDeadlinesIndexation() { - for (auto&& i : ActiveIndexationTasks) { - if (TMonotonic::Now() - i.second > NOlap::TCompactionLimits::CompactionTimeout) { - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "deadline_indexation")("task_id", i.first); - Y_DEBUG_ABORT_UNLESS(false); - } - } -} - -void TBackgroundController::StartIndexing(const NOlap::TColumnEngineChanges& changes) { - LastIndexationInstant = TMonotonic::Now(); - Y_ABORT_UNLESS(ActiveIndexationTasks.emplace(changes.GetTaskIdentifier(), TMonotonic::Now()).second); -} - -void TBackgroundController::FinishIndexing(const NOlap::TColumnEngineChanges& changes) { - Y_ABORT_UNLESS(ActiveIndexationTasks.erase(changes.GetTaskIdentifier())); -} - -TString TBackgroundController::DebugStringIndexation() const { - TStringBuilder sb; - sb << "{"; - sb << "task_ids="; - for (auto&& i : ActiveIndexationTasks) { - sb << i.first << ","; - } - sb << ";"; - sb << "}"; - return sb; -} - } diff --git a/ydb/core/tx/columnshard/background_controller.h b/ydb/core/tx/columnshard/background_controller.h index bb5e0eb3ad8f..e764ea251bf1 100644 --- a/ydb/core/tx/columnshard/background_controller.h +++ b/ydb/core/tx/columnshard/background_controller.h @@ -12,8 +12,6 @@ namespace NKikimr::NColumnShard { class TBackgroundController { private: - THashMap ActiveIndexationTasks; - using TCurrentCompaction = THashMap; TCurrentCompaction ActiveCompactionInfo; std::optional WaitingCompactionPriority; @@ -45,22 +43,14 @@ class TBackgroundController { } void CheckDeadlines(); - void CheckDeadlinesIndexation(); - bool StartCompaction(const TInternalPathId pathId); + bool StartCompaction(const TInternalPathId pathId, const TString& taskId); void FinishCompaction(const TInternalPathId pathId); ui32 GetCompactionsCount() const { return ActiveCompactionInfo.size(); } - void StartIndexing(const NOlap::TColumnEngineChanges& changes); - void FinishIndexing(const NOlap::TColumnEngineChanges& changes); - TString DebugStringIndexation() const; - i64 GetIndexingActiveCount() const { - return ActiveIndexationTasks.size(); - } - void StartCleanupPortions() { Y_ABORT_UNLESS(!ActiveCleanupPortions); ActiveCleanupPortions = true; diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h index fe99d6e8eb73..f42c3851871e 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h +++ b/ydb/core/tx/columnshard/engines/changes/abstract/compaction_info.h @@ -1,7 +1,9 @@ #pragma once +#include + +#include #include #include -#include #include #include @@ -17,6 +19,7 @@ class TPlanCompactionInfo { TInternalPathId PathId; TMonotonic StartTime = TMonotonic::Now(); TPositiveControlInteger Count; + YDB_READONLY_DEF(TString, TaskId); public: void Start() { @@ -30,8 +33,9 @@ class TPlanCompactionInfo { return StartTime; } - explicit TPlanCompactionInfo(const TInternalPathId pathId) - : PathId(pathId) { + explicit TPlanCompactionInfo(const TInternalPathId pathId, const TString& taskId) + : PathId(pathId) + , TaskId(taskId) { } TInternalPathId GetPathId() const { diff --git a/ydb/core/tx/columnshard/engines/changes/compaction.cpp b/ydb/core/tx/columnshard/engines/changes/compaction.cpp index 355c7a20b223..a9a2c05f653d 100644 --- a/ydb/core/tx/columnshard/engines/changes/compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/compaction.cpp @@ -34,7 +34,7 @@ void TCompactColumnEngineChanges::DoCompile(TFinalizationContext& context) { void TCompactColumnEngineChanges::DoStart(NColumnShard::TColumnShard& self) { TBase::DoStart(self); - self.BackgroundController.StartCompaction(GranuleMeta->GetPathId()); + self.BackgroundController.StartCompaction(GranuleMeta->GetPathId(), GetTaskIdentifier()); NeedGranuleStatusProvide = true; GranuleMeta->OnCompactionStarted(); } diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean_deprecated_snapshot.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean_deprecated_snapshot.cpp index 88148340c040..1cc4b871b4e8 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean_deprecated_snapshot.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean_deprecated_snapshot.cpp @@ -27,7 +27,7 @@ std::optional> GetChunksToRewrite( return std::nullopt; } } - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("tasks_for_rewrite", chunksToRewrite.size()); + AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("tasks_for_rewrite", chunksToRewrite.size()); return chunksToRewrite; } From d96eb41ab394e442390452de8ad93ebe588abefb Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Mon, 30 Jun 2025 18:05:28 +0300 Subject: [PATCH 45/62] special different fixes (#20369) Co-authored-by: ivanmorozov333 --- ydb/core/tx/columnshard/columnshard_impl.cpp | 3 +- .../engines/changes/abstract/abstract.cpp | 1 + .../engines/changes/general_compaction.cpp | 2 +- .../engines/portions/data_accessor.h | 31 +++++++-- .../reader/plain_reader/iterator/source.cpp | 5 +- .../reader/simple_reader/iterator/source.cpp | 5 +- ydb/core/tx/general_cache/service/manager.h | 66 +++++++++++-------- ydb/library/formats/arrow/splitter/stats.cpp | 2 +- 8 files changed, 78 insertions(+), 37 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index dd3a7db0912c..bd8cdb4eea70 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -611,8 +611,7 @@ class TCompactionExecutor: public NOlap::NDataFetcher::IFetchCallback { , Counters(counters) , SnapshotModification(snapshotModification) , NeedBlobs(needBlobs) - , TabletActivity(tabletActivity) - { + , TabletActivity(tabletActivity) { } }; diff --git a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp index f12fa3e99785..5cee27f00079 100644 --- a/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp +++ b/ydb/core/tx/columnshard/engines/changes/abstract/abstract.cpp @@ -10,6 +10,7 @@ namespace NKikimr::NOlap { void TColumnEngineChanges::SetStage(const NChanges::EStage stage) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "new_stage")("stage", ::ToString(stage))("task_id", GetTaskIdentifier()); StateGuard.SetState(stage); } diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index f57749aeda42..4cdda32cc717 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -59,7 +59,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc std::shared_ptr resultFiltered = context.BuildResultFiltered(accessors, seqDataColumnIds); std::shared_ptr stats = std::make_shared(); for (auto&& accessor : accessors) { - stats->Merge(accessor.GetSerializationStat(*resultFiltered)); + stats->Merge(accessor.GetSerializationStat(*resultFiltered, true)); } std::vector portions = TReadPortionInfoWithBlobs::RestorePortions(accessors, Blobs, context.SchemaVersions); diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index 5e827867662d..66e28fc0a5b7 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -410,12 +410,35 @@ class TPortionDataAccessor: public TPortionMetaBase { void RemoveFromDatabase(IDbWrapper& db) const; void SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const; - NArrow::NSplitter::TSerializationStats GetSerializationStat(const ISnapshotSchema& schema) const { + NArrow::NSplitter::TSerializationStats GetSerializationStat(const ISnapshotSchema& schema, const bool zeroIfAbsent = false) const { NArrow::NSplitter::TSerializationStats result; - for (auto&& i : GetRecordsVerified()) { - if (auto col = schema.GetFieldByColumnIdOptional(i.ColumnId)) { - result.AddStat(i.GetSerializationStat(col->name())); + if (!zeroIfAbsent) { + for (auto&& i : GetRecordsVerified()) { + if (auto col = schema.GetFieldByColumnIdOptional(i.ColumnId)) { + result.AddStat(i.GetSerializationStat(col->name())); + } + } + } else { + auto itSchema = schema.GetColumnIds().begin(); + auto itRecords = GetRecordsVerified().begin(); + while (itSchema != schema.GetColumnIds().end() && itRecords != GetRecordsVerified().end()) { + if (*itSchema < itRecords->ColumnId) { + NArrow::NSplitter::TColumnSerializationStat stat(*itSchema, schema.GetFieldByColumnIdVerified(*itSchema)->name()); + NArrow::NSplitter::TSimpleSerializationStat simpleStat(0, GetPortionInfo().GetRecordsCount(), 0); + stat.Merge(simpleStat); + result.AddStat(stat); + ++itSchema; + } else if (itRecords->ColumnId < *itSchema) { + ++itRecords; + } else { + while (itRecords != GetRecordsVerified().end() && itRecords->ColumnId == *itSchema) { + result.AddStat(itRecords->GetSerializationStat(schema.GetFieldByColumnIdVerified(*itSchema)->name())); + ++itRecords; + } + ++itSchema; + } } + } return result; } diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp index 89c83c3ee306..090961172ab1 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp @@ -169,7 +169,10 @@ class TPortionAccessorFetchingSubscriber: public IDataAccessorRequestsSubscriber return Source->GetContext()->GetCommonContext()->GetAbortionFlag(); } virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { - AFL_VERIFY(!result.HasErrors()); + if (result.HasErrors()) { + Source->GetContext()->GetCommonContext()->AbortWithError("has errors on portion accessors restore"); + return; + } AFL_VERIFY(result.GetPortions().size() == 1)("count", result.GetPortions().size()); Source->MutableStageData().SetPortionAccessor(std::move(result.ExtractPortionsVector().front())); AFL_VERIFY(Step.Next()); diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index 6262f5e50455..c8c331f3e882 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -385,7 +385,10 @@ class TPortionAccessorFetchingSubscriber: public IDataAccessorRequestsSubscriber virtual void DoOnRequestsFinished(TDataAccessorsResult&& result) override { FOR_DEBUG_LOG(NKikimrServices::COLUMNSHARD_SCAN_EVLOG, Source->AddEvent("facc")); - AFL_VERIFY(!result.HasErrors()); + if (result.HasErrors()) { + Source->GetContext()->GetCommonContext()->AbortWithError("has errors on portion accessors restore"); + return; + } AFL_VERIFY(result.GetPortions().size() == 1)("count", result.GetPortions().size()); Source->MutableStageData().SetPortionAccessor(std::move(result.ExtractPortionsVector().front())); Source->InitUsedRawBytes(); diff --git a/ydb/core/tx/general_cache/service/manager.h b/ydb/core/tx/general_cache/service/manager.h index 8eafcf4eb06d..3516bbb9f7f8 100644 --- a/ydb/core/tx/general_cache/service/manager.h +++ b/ydb/core/tx/general_cache/service/manager.h @@ -293,20 +293,28 @@ class TManager { return it->second; } - void AddObjects(const TSourceId sourceId, THashMap&& add, const bool isAdditional, const TMonotonic now) { + TSourceInfo* MutableSourceInfoOptional(const TSourceId sourceId) { + auto it = SourcesInfo.find(sourceId); + if (it != SourcesInfo.end()) { + return &it->second; + } + return nullptr; + } + + TSourceInfo& UpsertSourceInfo(const TSourceId sourceId) { + auto it = SourcesInfo.find(sourceId); + if (it == SourcesInfo.end()) { + it = SourcesInfo.emplace(sourceId, TSourceInfo(Counters, sourceId, ObjectsProcessor)).first; + } + return it->second; + } + + void AddObjectsToCache(const THashMap& add) { for (auto&& i : add) { Cache.Insert(i.first, i.second); } Counters->CacheSizeCount->Set(Cache.Size()); Counters->CacheSizeBytes->Set(Cache.TotalSize()); - if (isAdditional && !SourcesInfo.contains(sourceId)) { - SourcesInfo.emplace(sourceId, TSourceInfo(Counters, sourceId, ObjectsProcessor)); - } - MutableSourceInfo(sourceId).AddObjects(std::move(add), isAdditional, now); - } - - void RemoveObjects(const TSourceId sourceId, THashSet&& remove, const bool isAdditional, const TMonotonic now) { - MutableSourceInfo(sourceId).RemoveObjects(std::move(remove), isAdditional, now); } public: @@ -329,8 +337,8 @@ class TManager { } void AbortSource(const TSourceId sourceId) { - if (SourcesInfo.contains(sourceId)) { - MutableSourceInfo(sourceId).Abort(); + if (auto* sourceInfo = MutableSourceInfoOptional(sourceId)) { + sourceInfo->Abort(); SourcesInfo.erase(sourceId); } } @@ -365,12 +373,9 @@ class TManager { Counters->RequestCacheMiss->Inc(); } for (auto&& i : request->GetWaitBySource()) { - auto it = SourcesInfo.find(i.first); - if (it == SourcesInfo.end()) { - it = SourcesInfo.emplace(i.first, TSourceInfo(Counters, i.first, ObjectsProcessor)).first; - } - it->second.EnqueueRequest(request); - it->second.DrainQueue(); + auto& sourceInfo = UpsertSourceInfo(i.first); + sourceInfo.EnqueueRequest(request); + sourceInfo.DrainQueue(); } } @@ -378,8 +383,10 @@ class TManager { AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "objects_info"); const TMonotonic now = TMonotonic::Now(); const bool inFlightLimitBrokenBefore = !Counters->CheckTotalLimit(); - AddObjects(sourceId, std::move(add), true, now); - RemoveObjects(sourceId, std::move(remove), true, now); + AddObjectsToCache(add); + auto& sourceInfo = UpsertSourceInfo(sourceId); + sourceInfo.AddObjects(std::move(add), true, now); + sourceInfo.RemoveObjects(std::move(remove), true, now); const bool inFlightLimitBrokenAfter = !Counters->CheckTotalLimit(); if (inFlightLimitBrokenBefore && !inFlightLimitBrokenAfter) { DrainQueue(); @@ -389,18 +396,23 @@ class TManager { } void OnRequestResult( - const TSourceId sourceId, THashMap&& objects, THashSet&& removed, THashMap&& failed) { + const TSourceId sourceId, THashMap&& add, THashSet&& removed, THashMap&& failed) { AFL_DEBUG(NKikimrServices::GENERAL_CACHE)("event", "on_result"); const TMonotonic now = TMonotonic::Now(); const bool inFlightLimitBrokenBefore = !Counters->CheckTotalLimit(); - AddObjects(sourceId, std::move(objects), false, now); - RemoveObjects(sourceId, std::move(removed), false, now); - MutableSourceInfo(sourceId).FailObjects(std::move(failed), now); - const bool inFlightLimitBrokenAfter = !Counters->CheckTotalLimit(); - if (inFlightLimitBrokenBefore && !inFlightLimitBrokenAfter) { - DrainQueue(); + AddObjectsToCache(add); + if (auto* sourceInfo = MutableSourceInfoOptional(sourceId)) { + sourceInfo->AddObjects(std::move(add), false, now); + sourceInfo->RemoveObjects(std::move(removed), false, now); + sourceInfo->FailObjects(std::move(failed), now); + const bool inFlightLimitBrokenAfter = !Counters->CheckTotalLimit(); + if (inFlightLimitBrokenBefore && !inFlightLimitBrokenAfter) { + DrainQueue(); + } else { + DrainQueue(sourceId); + } } else { - DrainQueue(sourceId); + AFL_VERIFY(Counters->CheckTotalLimit() == !inFlightLimitBrokenBefore); } } }; diff --git a/ydb/library/formats/arrow/splitter/stats.cpp b/ydb/library/formats/arrow/splitter/stats.cpp index 63f3ad468409..b85bd514b297 100644 --- a/ydb/library/formats/arrow/splitter/stats.cpp +++ b/ydb/library/formats/arrow/splitter/stats.cpp @@ -27,7 +27,7 @@ TSimpleSerializationStat::TSimpleSerializationStat(const ui64 bytes, const ui64 : SerializedBytes(bytes) , RecordsCount(recordsCount) , RawBytes(rawBytes) { - Y_ABORT_UNLESS(SerializedBytes); +// Y_ABORT_UNLESS(SerializedBytes); Y_ABORT_UNLESS(RecordsCount); // Y_ABORT_UNLESS(RawBytes); } From b6909b15235afe15f6fb56d0319367f292fcf066 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 1 Jul 2025 10:31:38 +0300 Subject: [PATCH 46/62] fix cursors usage (#20394) Co-authored-by: ivanmorozov333 --- ydb/core/kqp/compute_actor/kqp_compute_state.h | 7 +++++++ .../kqp/compute_actor/kqp_scan_compute_manager.h | 4 ++-- .../kqp/compute_actor/kqp_scan_fetcher_actor.cpp | 13 ++++++++----- .../kqp/compute_actor/kqp_scan_fetcher_actor.h | 3 +++ ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 13 ++++++------- ydb/core/testlib/test_client.h | 4 ++++ .../columnshard/engines/portions/data_accessor.h | 7 +++++-- .../iterator/collections/full_scan_sorted.h | 15 ++++++++++----- .../iterator/collections/limit_sorted.cpp | 3 +++ .../iterator/collections/not_sorted.h | 2 ++ .../tools/kqprun/configuration/app_config.conf | 1 + 11 files changed, 51 insertions(+), 21 deletions(-) diff --git a/ydb/core/kqp/compute_actor/kqp_compute_state.h b/ydb/core/kqp/compute_actor/kqp_compute_state.h index b4b71f3b7262..e282d78e5cab 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_state.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_state.h @@ -41,6 +41,13 @@ struct TShardState: public TCommonRetriesState { std::optional LastCursorProto; std::optional AvailablePacks; + TString CursorDebugString() const { + TString strCursor = LastCursorProto ? LastCursorProto->DebugString() : TString("START"); + while (strCursor.find("\n") != std::string::npos) { + strCursor.replace(strCursor.find("\n"), 1, " "); + } + return strCursor; + } TString PrintLastKey(TConstArrayRef keyTypes) const; TShardState(const ui64 tabletId); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h index bcc0e28fe4ce..5114ab963299 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h @@ -66,8 +66,8 @@ class TShardScannerInfo { auto ev = externalObjectsProvider.BuildEvKqpScan(ScanId, Generation, ranges, state.LastCursorProto); AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner")("tablet_id", TabletId)("generation", Generation) - ("info", state.ToString(keyColumnTypes))("range", DebugPrintRanges(keyColumnTypes, ranges, *AppData()->TypeRegistry)) - ("subscribed", subscribed); + ("info", state.ToString(keyColumnTypes))("range", DebugPrintRanges(keyColumnTypes, ranges, *AppData()->TypeRegistry))( + "subscribed", subscribed)("cursor", state.CursorDebugString()); NActors::TActivationContext::AsActorContext().Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(ev.release(), TabletId, !subscribed), IEventHandle::FlagTrackDelivery); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp index afde096a3485..569c35b993b6 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp @@ -44,7 +44,8 @@ TKqpScanFetcherActor::TKqpScanFetcherActor(const NKikimrKqp::TKqpSnapshot& snaps , ShardsScanningPolicy(shardsScanningPolicy) , Counters(counters) , InFlightShards(ScanId, *this) - , InFlightComputes(ComputeActorIds) { + , InFlightComputes(ComputeActorIds) + , IsOlapTable(Meta.GetTable().GetTableKind() == (ui32)ETableKind::Olap) { Y_UNUSED(traceId); AFL_ENSURE(!Meta.GetReads().empty()); AFL_ENSURE(Meta.GetTable().GetTableKind() != (ui32)ETableKind::SysView); @@ -86,7 +87,8 @@ void TKqpScanFetcherActor::Bootstrap() { for (auto&& c : ComputeActorIds) { Sender().SendTo(c); } - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "bootstrap")("compute", ComputeActorIds.size())("shards", PendingShards.size()); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "bootstrap")("self_if", SelfId())("compute", ComputeActorIds.size())( + "shards", PendingShards.size()); StartTableScan(); Become(&TKqpScanFetcherActor::StateFunc); Schedule(PING_PERIOD, new NActors::TEvents::TEvWakeup()); @@ -301,7 +303,7 @@ void TKqpScanFetcherActor::HandleExecute(TEvTxProxySchemeCache::TEvResolveKeySet } const auto& tr = *AppData()->TypeRegistry; - if (Meta.HasOlapProgram()) { + if (IsOlapTable) { bool found = false; for (auto&& partition : keyDesc->GetPartitions()) { if (partition.ShardId != state.TabletId) { @@ -534,8 +536,9 @@ void TKqpScanFetcherActor::ProcessPendingScanDataItem(TEvKqpCompute::TEvScanData state->LastCursorProto = std::move(msg.LastCursorProto); const ui64 rowsCount = msg.GetRowsCount(); AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("action", "got EvScanData")("rows", rowsCount)("finished", msg.Finished)( - "exceeded", msg.RequestedBytesLimitReached)("scan", ScanId)("packs_to_send", InFlightComputes.GetPacksToSendCount())("from", ev->Sender)( - "shards remain", PendingShards.size())("in flight scans", InFlightShards.GetScansCount())( + "generation", msg.Generation)("exceeded", msg.RequestedBytesLimitReached)("scan", ScanId)( + "packs_to_send", InFlightComputes.GetPacksToSendCount())("from", ev->Sender)("shards remain", PendingShards.size())( + "in flight scans", InFlightShards.GetScansCount())("cursor", state->CursorDebugString())( "in flight shards", InFlightShards.GetShardsCount())("delayed_for_seconds_by_ratelimiter", latency.SecondsFloat())( "tablet_id", state->TabletId)("locks", msg.LocksInfo.Locks.size())("broken locks", msg.LocksInfo.BrokenLocks.size()); auto shardScanner = InFlightShards.GetShardScannerVerified(state->TabletId); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h index 90b8bc29299d..5ef0a9d82c10 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h @@ -72,6 +72,8 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped(TxId)); try { switch (ev->GetTypeRewrite()) { hFunc(TEvKqpCompute::TEvScanInitActor, HandleExecute); @@ -184,6 +186,7 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped TrackingNodes; diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index ad2d09dd37c0..aa4fc7a8acc6 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1729,7 +1729,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { auto settings = Tests::TServerSettings(mbusport) .SetDomainName("Root") .SetUseRealThreads(false) - .SetNodeCount(2); + .SetNodeCount(2).SetColumnShardReaderClassName("SIMPLE"); Tests::TServer::TPtr server = new Tests::TServer(settings); @@ -1814,8 +1814,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { ui16 mbusport = tp.GetPort(2134); auto settings = Tests::TServerSettings(mbusport) .SetDomainName("Root") - .SetUseRealThreads(false) - .SetNodeCount(2); + .SetUseRealThreads(false).SetNodeCount(2).SetColumnShardReaderClassName("SIMPLE"); Tests::TServer::TPtr server = new Tests::TServer(settings); @@ -1881,8 +1880,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { ui16 mbusport = tp.GetPort(2134); auto settings = Tests::TServerSettings(mbusport) .SetDomainName("Root") - .SetUseRealThreads(false) - .SetNodeCount(2); + .SetUseRealThreads(false).SetNodeCount(2).SetColumnShardReaderClassName( + "SIMPLE"); Tests::TServer::TPtr server = new Tests::TServer(settings); @@ -1943,8 +1942,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { ui16 mbusport = tp.GetPort(2134); auto settings = Tests::TServerSettings(mbusport) .SetDomainName("Root") - .SetUseRealThreads(false) - .SetNodeCount(2); + .SetUseRealThreads(false).SetNodeCount(2).SetColumnShardReaderClassName( + "SIMPLE"); Tests::TServer::TPtr server = new Tests::TServer(settings); auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index 969fcff2e782..f2de654f6c5a 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -229,6 +229,10 @@ namespace Tests { TServerSettings& SetInitializeFederatedQuerySetupFactory(bool value) { InitializeFederatedQuerySetupFactory = value; return *this; } TServerSettings& SetVerbose(bool value) { Verbose = value; return *this; } TServerSettings& SetUseSectorMap(bool value) { UseSectorMap = value; return *this; } + TServerSettings& SetColumnShardReaderClassName(const TString& className) { + AppConfig->MutableColumnShardConfig()->SetReaderClassName(className); + return *this; + } TServerSettings& SetPersQueueGetReadSessionsInfoWorkerFactory( std::shared_ptr factory ) { diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.h b/ydb/core/tx/columnshard/engines/portions/data_accessor.h index 66e28fc0a5b7..2ef3414caf50 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.h +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.h @@ -421,21 +421,24 @@ class TPortionDataAccessor: public TPortionMetaBase { } else { auto itSchema = schema.GetColumnIds().begin(); auto itRecords = GetRecordsVerified().begin(); + ui32 schemaIdx = 0; while (itSchema != schema.GetColumnIds().end() && itRecords != GetRecordsVerified().end()) { if (*itSchema < itRecords->ColumnId) { - NArrow::NSplitter::TColumnSerializationStat stat(*itSchema, schema.GetFieldByColumnIdVerified(*itSchema)->name()); + NArrow::NSplitter::TColumnSerializationStat stat(*itSchema, schema.GetFieldByIndexVerified(schemaIdx)->name()); NArrow::NSplitter::TSimpleSerializationStat simpleStat(0, GetPortionInfo().GetRecordsCount(), 0); stat.Merge(simpleStat); result.AddStat(stat); ++itSchema; + ++schemaIdx; } else if (itRecords->ColumnId < *itSchema) { ++itRecords; } else { while (itRecords != GetRecordsVerified().end() && itRecords->ColumnId == *itSchema) { - result.AddStat(itRecords->GetSerializationStat(schema.GetFieldByColumnIdVerified(*itSchema)->name())); + result.AddStat(itRecords->GetSerializationStat(schema.GetFieldByIndexVerified(schemaIdx)->name())); ++itRecords; } ++itSchema; + ++schemaIdx; } } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/full_scan_sorted.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/full_scan_sorted.h index 8c4d960fd69d..709ac500f312 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/full_scan_sorted.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/full_scan_sorted.h @@ -45,20 +45,25 @@ class TSortedFullScanCollection: public ISourcesCollection { TSortedFullScanCollection(const std::shared_ptr& context, std::deque&& sources, const std::shared_ptr& cursor) : TBase(context) { + HeapSources = std::move(sources); + std::make_heap(HeapSources.begin(), HeapSources.end()); if (cursor && cursor->IsInitialized()) { - for (auto&& i : sources) { + while (HeapSources.size()) { bool usage = false; - if (!context->GetCommonContext()->GetScanCursor()->CheckEntityIsBorder(i, usage)) { + if (!context->GetCommonContext()->GetScanCursor()->CheckEntityIsBorder(HeapSources.front(), usage)) { + std::pop_heap(HeapSources.begin(), HeapSources.end()); + HeapSources.pop_back(); continue; } if (usage) { - i.SetIsStartedByCursor(); + HeapSources.front().SetIsStartedByCursor(); + } else { + std::pop_heap(HeapSources.begin(), HeapSources.end()); + HeapSources.pop_back(); } break; } } - HeapSources = std::move(sources); - std::make_heap(HeapSources.begin(), HeapSources.end()); } }; diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/limit_sorted.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/limit_sorted.cpp index 33b2c050e534..c4dff941ab09 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/limit_sorted.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/limit_sorted.cpp @@ -42,6 +42,9 @@ TScanWithLimitCollection::TScanWithLimitCollection( } if (usage) { HeapSources.front().SetIsStartedByCursor(); + } else { + std::pop_heap(HeapSources.begin(), HeapSources.end()); + HeapSources.pop_back(); } break; } diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/not_sorted.h b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/not_sorted.h index 45b855d9db0e..7d87febdce71 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/not_sorted.h +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/collections/not_sorted.h @@ -67,6 +67,8 @@ class TNotSortedCollection: public ISourcesCollection { } if (usage) { sources.front().SetIsStartedByCursor(); + } else { + sources.pop_front(); } break; } diff --git a/ydb/tests/tools/kqprun/configuration/app_config.conf b/ydb/tests/tools/kqprun/configuration/app_config.conf index b134f0daced4..0a136a76599d 100644 --- a/ydb/tests/tools/kqprun/configuration/app_config.conf +++ b/ydb/tests/tools/kqprun/configuration/app_config.conf @@ -47,6 +47,7 @@ ActorSystemConfig { ColumnShardConfig { DisabledOnSchemeShard: false WritingInFlightRequestBytesLimit: 104857600 + ReaderClassName: "SIMPLE" } FeatureFlags { From 96262dd806b7aa7dcd59d5ca69ce0d34d8d0d9cf Mon Sep 17 00:00:00 2001 From: Vladilen Date: Tue, 1 Jul 2025 10:26:51 +0200 Subject: [PATCH 47/62] Fix bulk CS stats (#20253) --- ydb/core/protos/counters_columnshard.proto | 2 ++ ydb/core/protos/data_events.proto | 1 + ydb/core/protos/tx_columnshard.proto | 1 + ydb/core/tablet/tablet_counters_aggregator.cpp | 8 ++++++-- .../tx/columnshard/blobs_action/abstract/write.h | 2 ++ .../tx/columnshard/blobs_action/bs/write.cpp | 6 +++++- ydb/core/tx/columnshard/columnshard__write.cpp | 16 +++++++++++++--- .../tx/columnshard/counters/counters_manager.h | 10 ++++++++++ .../tx/columnshard/counters/tablet_counters.h | 12 +++++++++++- ydb/core/tx/columnshard/operations/manager.cpp | 6 +++--- ydb/core/tx/columnshard/operations/manager.h | 2 +- ydb/core/tx/columnshard/operations/write.cpp | 14 ++++++++++---- ydb/core/tx/columnshard/operations/write.h | 3 ++- ydb/core/tx/columnshard/tablet/write_queue.cpp | 2 +- ydb/core/tx/columnshard/tablet/write_queue.h | 4 +++- ydb/core/tx/data_events/columnshard_splitter.h | 1 + ydb/core/tx/data_events/write_data.h | 2 ++ 17 files changed, 74 insertions(+), 18 deletions(-) diff --git a/ydb/core/protos/counters_columnshard.proto b/ydb/core/protos/counters_columnshard.proto index d1c7a6d5e683..dd35e3d95df3 100644 --- a/ydb/core/protos/counters_columnshard.proto +++ b/ydb/core/protos/counters_columnshard.proto @@ -139,6 +139,8 @@ enum ECumulativeCounters { COUNTER_IMMEDIATE_TX_COMPLETED = 83 [(CounterOpts) = {Name: "ImmediateTxCompleted"}]; COUNTER_ROWS_ERASED = 84 [(CounterOpts) = {Name: "RowsErased"}]; COUNTER_OPERATIONS_ROWS_WRITTEN = 85 [(CounterOpts) = {Name: "OperationsRowsWritten"}]; + COUNTER_OPERATIONS_BULK_ROWS_WRITTEN = 86 [(CounterOpts) = {Name: "OperationsBulkRowsWritten"}]; + COUNTER_OPERATIONS_BULK_BYTES_WRITTEN = 87 [(CounterOpts) = {Name: "OperationsBulkBytesWritten"}]; } enum EPercentileCounters { diff --git a/ydb/core/protos/data_events.proto b/ydb/core/protos/data_events.proto index 9a28e8843137..6817cb938520 100644 --- a/ydb/core/protos/data_events.proto +++ b/ydb/core/protos/data_events.proto @@ -102,6 +102,7 @@ message TEvWrite { optional uint64 PayloadIndex = 4; optional EDataFormat PayloadFormat = 5; optional string PayloadSchema = 6; + optional bool IsBulk = 8; } // Transaction operations diff --git a/ydb/core/protos/tx_columnshard.proto b/ydb/core/protos/tx_columnshard.proto index 0893d4cd3ca8..afdd132c1ad5 100644 --- a/ydb/core/protos/tx_columnshard.proto +++ b/ydb/core/protos/tx_columnshard.proto @@ -354,4 +354,5 @@ message TInternalOperationData { optional uint32 ModificationType = 2; optional uint64 PathId = 3; optional bool WritePortions = 4; + optional bool IsBulk = 5; } diff --git a/ydb/core/tablet/tablet_counters_aggregator.cpp b/ydb/core/tablet/tablet_counters_aggregator.cpp index 3b06bb4c0954..95e0e2ab29b8 100644 --- a/ydb/core/tablet/tablet_counters_aggregator.cpp +++ b/ydb/core/tablet/tablet_counters_aggregator.cpp @@ -817,6 +817,8 @@ class TTabletMon { TCounterPtr ColumnShardScannedRows_; TCounterPtr ColumnShardOperationsRowsWritten_; TCounterPtr ColumnShardOperationsBytesWritten_; + TCounterPtr ColumnShardOperationsBulkRowsWritten_; + TCounterPtr ColumnShardOperationsBulkBytesWritten_; TCounterPtr ColumnShardErasedBytes_; TCounterPtr ColumnShardErasedRows_; THistogramPtr ColumnShardConsumedCpuHistogram; @@ -969,6 +971,8 @@ class TTabletMon { ColumnShardScannedRows_ = appGroup->GetCounter("ColumnShard/ScannedRows"); ColumnShardOperationsRowsWritten_ = appGroup->GetCounter("ColumnShard/OperationsRowsWritten"); ColumnShardOperationsBytesWritten_ = appGroup->GetCounter("ColumnShard/OperationsBytesWritten"); + ColumnShardOperationsBulkRowsWritten_ = appGroup->GetCounter("ColumnShard/OperationsBulkRowsWritten"); + ColumnShardOperationsBulkBytesWritten_ = appGroup->GetCounter("ColumnShard/OperationsBulkBytesWritten"); ColumnShardErasedBytes_ = appGroup->GetCounter("ColumnShard/BytesErased"); ColumnShardErasedRows_ = appGroup->GetCounter("ColumnShard/RowsErased"); ColumnShardConsumedCpuHistogram = appGroup->FindHistogram("HIST(ConsumedCPU)"); @@ -1023,8 +1027,8 @@ class TTabletMon { ColumnShardScanBytes_->Set(ColumnShardScannedBytes_->Val()); ColumnShardWriteRows_->Set(ColumnShardOperationsRowsWritten_->Val()); ColumnShardWriteBytes_->Set(ColumnShardOperationsBytesWritten_->Val()); - ColumnShardBulkUpsertRows_->Set(ColumnShardOperationsRowsWritten_->Val()); - ColumnShardBulkUpsertBytes_->Set(ColumnShardOperationsBytesWritten_->Val()); + ColumnShardBulkUpsertRows_->Set(ColumnShardOperationsBulkRowsWritten_->Val()); + ColumnShardBulkUpsertBytes_->Set(ColumnShardOperationsBulkBytesWritten_->Val()); ColumnShardEraseRows_->Set(ColumnShardErasedRows_->Val()); ColumnShardEraseBytes_->Set(ColumnShardErasedBytes_->Val()); diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/write.h b/ydb/core/tx/columnshard/blobs_action/abstract/write.h index f20d4183cefb..a67fe3a0a47d 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/write.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/write.h @@ -23,7 +23,9 @@ class IBlobsWritingAction: public ICommonBlobsAction { THashSet BlobsWaiting; bool Aborted = false; std::shared_ptr Counters; + YDB_FLAG_ACCESSOR(Bulk, false); void AddDataForWrite(const TUnifiedBlobId& blobId, const TString& data); + protected: virtual void DoOnExecuteTxBeforeWrite(NColumnShard::TColumnShard& self, TBlobManagerDb& dbBlobs) = 0; virtual void DoOnCompleteTxBeforeWrite(NColumnShard::TColumnShard& self) = 0; diff --git a/ydb/core/tx/columnshard/blobs_action/bs/write.cpp b/ydb/core/tx/columnshard/blobs_action/bs/write.cpp index 06ebc64eccf3..628510f63055 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/write.cpp @@ -15,7 +15,11 @@ void TWriteAction::DoOnCompleteTxAfterWrite(NColumnShard::TColumnShard& self, co ui64 blobsWritten = BlobBatch.GetBlobCount(); ui64 bytesWritten = BlobBatch.GetTotalSize(); if (blobsWroteSuccessfully) { - self.Counters.GetTabletCounters()->OnWriteSuccess(blobsWritten, bytesWritten); + if (IsBulk()) { + self.Counters.GetTabletCounters()->OnBulkWriteSuccess(blobsWritten, bytesWritten); + } else { + self.Counters.GetTabletCounters()->OnWriteSuccess(blobsWritten, bytesWritten); + } Manager->SaveBlobBatchOnComplete(std::move(BlobBatch)); } else { self.Counters.GetTabletCounters()->OnWriteFailure(); diff --git a/ydb/core/tx/columnshard/columnshard__write.cpp b/ydb/core/tx/columnshard/columnshard__write.cpp index 081e05a49973..b51f453aeb10 100644 --- a/ydb/core/tx/columnshard/columnshard__write.cpp +++ b/ydb/core/tx/columnshard/columnshard__write.cpp @@ -105,7 +105,11 @@ void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& e AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", i.GetDataSize())("event", "data_write_finished")( "writing_id", i.GetWriteMeta().GetId()); i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::SuccessWritingToLocalDB); - Counters.OnWritePutBlobsSuccess(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + if (i.GetWriteMeta().IsBulk()) { + Counters.OnWritePutBulkBlobsSuccess(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + } else { + Counters.OnWritePutBlobsSuccess(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + } Counters.GetWritesMonitor()->OnFinishWrite(i.GetDataSize(), 1); } Execute(new TTxBlobsWritingFinished(this, ev->Get()->GetWriteStatus(), ev->Get()->GetWriteAction(), std::move(writtenData)), ctx); @@ -113,7 +117,11 @@ void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& e const TMonotonic now = TMonotonic::Now(); for (auto&& i : writtenData.GetWriteResults()) { i.MutableWriteMeta().OnStage(NEvWrite::EWriteStage::FailWritingToLocalDB); - Counters.OnWritePutBlobsFailed(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + if (i.GetWriteMeta().IsBulk()) { + Counters.OnWritePutBulkBlobsFailed(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + } else { + Counters.OnWritePutBlobsFailed(now - i.GetWriteMeta().GetWriteStartInstant(), i.GetRecordsCount()); + } Counters.GetCSCounters().OnWritePutBlobsFail(now - i.GetWriteMeta().GetWriteStartInstant()); AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", i.GetDataSize())("event", "data_write_error")( "writing_id", i.GetWriteMeta().GetId())("reason", i.GetErrorMessage()); @@ -477,9 +485,11 @@ void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActor lockId = record.GetLockTxId(); } + const bool isBulk = operation.HasIsBulk() && operation.GetIsBulk(); + Counters.GetWritesMonitor()->OnStartWrite(arrowData->GetSize()); WriteTasksQueue->Enqueue(TWriteTask( - arrowData, schema, source, granuleShardingVersionId, pathId, cookie, lockId, *mType, behaviour, writeTimeout, record.GetTxId())); + arrowData, schema, source, granuleShardingVersionId, pathId, cookie, lockId, *mType, behaviour, writeTimeout, record.GetTxId(), isBulk)); WriteTasksQueue->Drain(false, ctx); } diff --git a/ydb/core/tx/columnshard/counters/counters_manager.h b/ydb/core/tx/columnshard/counters/counters_manager.h index 8cf93aa0caaf..4e9db8eafe04 100644 --- a/ydb/core/tx/columnshard/counters/counters_manager.h +++ b/ydb/core/tx/columnshard/counters/counters_manager.h @@ -98,10 +98,20 @@ class TCountersManager { CSCounters.OnWritePutBlobsSuccess(d); } + void OnWritePutBulkBlobsSuccess(const TDuration d, const ui64 rowsWritten) const { + TabletCounters->OnWritePutBulkBlobsSuccess(rowsWritten); + CSCounters.OnWritePutBlobsSuccess(d); + } + void OnWritePutBlobsFailed(const TDuration d, const ui64 /*rowsWritten*/) const { TabletCounters->OnWriteFailure(); CSCounters.OnWritePutBlobsFail(d); } + + void OnWritePutBulkBlobsFailed(const TDuration d, const ui64 /*rowsWritten*/) const { + TabletCounters->OnWriteFailure(); + CSCounters.OnWritePutBlobsFail(d); + } }; } // namespace NKikimr::NColumnShard diff --git a/ydb/core/tx/columnshard/counters/tablet_counters.h b/ydb/core/tx/columnshard/counters/tablet_counters.h index 8172278cf1bf..585bd2fff86a 100644 --- a/ydb/core/tx/columnshard/counters/tablet_counters.h +++ b/ydb/core/tx/columnshard/counters/tablet_counters.h @@ -75,6 +75,12 @@ class TTabletCountersHandle { IncCounter(NColumnShard::COUNTER_WRITE_SUCCESS); } + void OnBulkWriteSuccess(const ui64 blobsWritten, const ui64 bytesWritten) const { + IncCounter(NColumnShard::COUNTER_OPERATIONS_BLOBS_WRITTEN, blobsWritten); + IncCounter(NColumnShard::COUNTER_OPERATIONS_BULK_BYTES_WRITTEN, bytesWritten); + IncCounter(NColumnShard::COUNTER_WRITE_SUCCESS); + } + void OnWriteFailure() const { IncCounter(NColumnShard::COUNTER_WRITE_FAIL); } @@ -96,6 +102,10 @@ class TTabletCountersHandle { IncCounter(NColumnShard::COUNTER_OPERATIONS_ROWS_WRITTEN, rowsWritten); } + void OnWritePutBulkBlobsSuccess(const ui64 rowsWritten) const { + IncCounter(NColumnShard::COUNTER_OPERATIONS_BULK_ROWS_WRITTEN, rowsWritten); + } + void OnDropPortionEvent(const ui64 rawBytes, const ui64 blobBytes, const ui64 rows) const { IncCounter(NColumnShard::COUNTER_RAW_BYTES_ERASED, rawBytes); IncCounter(NColumnShard::COUNTER_BYTES_ERASED, blobBytes); @@ -103,7 +113,7 @@ class TTabletCountersHandle { } void FillStats(::NKikimrTableStats::TTableStats& output) const { - output.SetRowUpdates(GetValue(COUNTER_OPERATIONS_ROWS_WRITTEN)); + output.SetRowUpdates(GetValue(COUNTER_OPERATIONS_ROWS_WRITTEN) + GetValue(COUNTER_OPERATIONS_BULK_ROWS_WRITTEN)); output.SetRowDeletes(GetValue(COUNTER_ROWS_ERASED)); output.SetRowReads(0); // all reads are range reads output.SetRangeReadRows(GetValue(COUNTER_READ_INDEX_ROWS)); diff --git a/ydb/core/tx/columnshard/operations/manager.cpp b/ydb/core/tx/columnshard/operations/manager.cpp index 362a2386a249..606b1e4fe069 100644 --- a/ydb/core/tx/columnshard/operations/manager.cpp +++ b/ydb/core/tx/columnshard/operations/manager.cpp @@ -29,7 +29,7 @@ bool TOperationsManager::Load(NTabletFlatExecutor::TTransactionContext& txc) { Y_ABORT_UNLESS(metaProto.ParseFromString(metadata)); auto operation = std::make_shared(TUnifiedPathId{}, writeId, lockId, cookie, status, TInstant::Seconds(createdAtSec), - granuleShardingVersionId, NEvWrite::EModificationType::Upsert); + granuleShardingVersionId, NEvWrite::EModificationType::Upsert, metaProto.GetIsBulk()); operation->FromProto(metaProto); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_TX)("event", "register_operation_on_load")("operation_id", operation->GetWriteId()); AFL_VERIFY(operation->GetStatus() != EOperationStatus::Draft); @@ -203,10 +203,10 @@ void TOperationsManager::LinkTransactionOnComplete(const ui64 /*lockId*/, const } TWriteOperation::TPtr TOperationsManager::CreateWriteOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, - const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType) { + const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool isBulk) { auto writeId = BuildNextOperationWriteId(); auto operation = std::make_shared(pathId, writeId, lockId, cookie, EOperationStatus::Draft, AppData()->TimeProvider->Now(), - granuleShardingVersionId, mType); + granuleShardingVersionId, mType, isBulk); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "register_operation")("operation_id", operation->GetWriteId())( "last", LastWriteId); AFL_VERIFY(Operations.emplace(operation->GetWriteId(), operation).second); diff --git a/ydb/core/tx/columnshard/operations/manager.h b/ydb/core/tx/columnshard/operations/manager.h index bb2f8d8f9eaa..e0c96a6b597c 100644 --- a/ydb/core/tx/columnshard/operations/manager.h +++ b/ydb/core/tx/columnshard/operations/manager.h @@ -208,7 +208,7 @@ class TOperationsManager { } TWriteOperation::TPtr CreateWriteOperation(const TUnifiedPathId& pathId, const ui64 lockId, const ui64 cookie, const std::optional granuleShardingVersionId, - const NEvWrite::EModificationType mType); + const NEvWrite::EModificationType mType, const bool isBulk); bool RegisterLock(const ui64 lockId, const ui64 generationId) { if (LockFeatures.contains(lockId)) { return false; diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 6318a303830f..2a92bd28fb76 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -16,7 +16,7 @@ namespace NKikimr::NColumnShard { TWriteOperation::TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, const TInstant createdAt, const std::optional granuleShardingVersionId, - const NEvWrite::EModificationType mType) + const NEvWrite::EModificationType mType, const bool isBulk) : PathId(pathId) , Status(status) , CreatedAt(createdAt) @@ -24,7 +24,9 @@ TWriteOperation::TWriteOperation(const TUnifiedPathId& pathId, const TOperationW , LockId(lockId) , Cookie(cookie) , GranuleShardingVersionId(granuleShardingVersionId) - , ModificationType(mType) { + , ModificationType(mType) + , BulkFlag(isBulk) +{ } void TWriteOperation::Start( @@ -36,8 +38,10 @@ void TWriteOperation::Start( context.GetWritingCounters()->GetWriteFlowCounters()); writeMeta->SetLockId(LockId); writeMeta->SetModificationType(ModificationType); - NEvWrite::TWriteData writeData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), - owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR)); + writeMeta->SetBulk(IsBulk()); + auto writingAction = owner.StoragesManager->GetInsertOperator()->StartWritingAction(NOlap::NBlobOperations::EConsumer::WRITING_OPERATOR); + writingAction->SetBulk(IsBulk()); + NEvWrite::TWriteData writeData(writeMeta, data, owner.TablesManager.GetPrimaryIndex()->GetReplaceKey(), std::move(writingAction)); std::shared_ptr task = std::make_shared(std::move(writeData), context); NConveyorComposite::TInsertServiceOperator::SendTaskToExecute(task); @@ -95,6 +99,7 @@ void TWriteOperation::ToProto(NKikimrTxColumnShard::TInternalOperationData& prot } proto.SetModificationType((ui32)ModificationType); proto.SetWritePortions(true); + proto.SetIsBulk(IsBulk()); PathId.InternalPathId.ToProto(proto); } @@ -109,6 +114,7 @@ void TWriteOperation::FromProto(const NKikimrTxColumnShard::TInternalOperationDa } else { ModificationType = NEvWrite::EModificationType::Replace; } + BulkFlag = proto.GetIsBulk(); } void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::TTransactionContext& txc) const { diff --git a/ydb/core/tx/columnshard/operations/write.h b/ydb/core/tx/columnshard/operations/write.h index 0eb1cf172c67..c01379562dd4 100644 --- a/ydb/core/tx/columnshard/operations/write.h +++ b/ydb/core/tx/columnshard/operations/write.h @@ -60,6 +60,7 @@ class TWriteOperation: public TMonitoringObjectsCounter { YDB_ACCESSOR(EOperationBehaviour, Behaviour, EOperationBehaviour::Undefined); YDB_READONLY_DEF(std::optional, GranuleShardingVersionId); YDB_READONLY(NEvWrite::EModificationType, ModificationType, NEvWrite::EModificationType::Upsert); + YDB_READONLY_FLAG(Bulk, false); const std::shared_ptr Activity = std::make_shared(1); public: @@ -70,7 +71,7 @@ class TWriteOperation: public TMonitoringObjectsCounter { } TWriteOperation(const TUnifiedPathId& pathId, const TOperationWriteId writeId, const ui64 lockId, const ui64 cookie, const EOperationStatus& status, - const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType); + const TInstant createdAt, const std::optional granuleShardingVersionId, const NEvWrite::EModificationType mType, const bool isBulk); void Start( TColumnShard& owner, const NEvWrite::IDataContainer::TPtr& data, const NActors::TActorId& source, const NOlap::TWritingContext& context); diff --git a/ydb/core/tx/columnshard/tablet/write_queue.cpp b/ydb/core/tx/columnshard/tablet/write_queue.cpp index fa9bc3255c6a..455a86b5fd4f 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.cpp +++ b/ydb/core/tx/columnshard/tablet/write_queue.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NColumnShard { bool TWriteTask::Execute(TColumnShard* owner, const TActorContext& /* ctx */) const { owner->Counters.GetCSCounters().WritingCounters->OnWritingTaskDequeue(TMonotonic::Now() - Created); owner->OperationsManager->RegisterLock(LockId, owner->Generation()); - auto writeOperation = owner->OperationsManager->CreateWriteOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType); + auto writeOperation = owner->OperationsManager->CreateWriteOperation(PathId, LockId, Cookie, GranuleShardingVersionId, ModificationType, IsBulk); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_WRITE)("writing_size", ArrowData->GetSize())("operation_id", writeOperation->GetIdentifier())( "in_flight", owner->Counters.GetWritesMonitor()->GetWritesInFlight())( diff --git a/ydb/core/tx/columnshard/tablet/write_queue.h b/ydb/core/tx/columnshard/tablet/write_queue.h index 0646d8edf33e..53de54141759 100644 --- a/ydb/core/tx/columnshard/tablet/write_queue.h +++ b/ydb/core/tx/columnshard/tablet/write_queue.h @@ -23,6 +23,7 @@ class TWriteTask: public TMoveOnly { const TMonotonic Created = TMonotonic::Now(); const std::optional Timeout; const ui64 TxId; + const bool IsBulk; public: bool operator<(const TWriteTask& item) const { @@ -35,7 +36,7 @@ class TWriteTask: public TMoveOnly { TWriteTask(const std::shared_ptr& arrowData, const NOlap::ISnapshotSchema::TPtr& schema, const NActors::TActorId sourceId, const std::optional& granuleShardingVersionId, const TUnifiedPathId pathId, const ui64 cookie, const ui64 lockId, - const NEvWrite::EModificationType modificationType, const EOperationBehaviour behaviour, const std::optional timeout, const ui64 txId) + const NEvWrite::EModificationType modificationType, const EOperationBehaviour behaviour, const std::optional timeout, const ui64 txId, const bool isBulk) : ArrowData(arrowData) , Schema(schema) , SourceId(sourceId) @@ -47,6 +48,7 @@ class TWriteTask: public TMoveOnly { , Behaviour(behaviour) , Timeout(timeout) , TxId(txId) + , IsBulk(isBulk) { } diff --git a/ydb/core/tx/data_events/columnshard_splitter.h b/ydb/core/tx/data_events/columnshard_splitter.h index cb699fe1ce98..798f36baca04 100644 --- a/ydb/core/tx/data_events/columnshard_splitter.h +++ b/ydb/core/tx/data_events/columnshard_splitter.h @@ -50,6 +50,7 @@ class TColumnShardShardsSplitter: public IShardsSplitter { operation->SetPayloadIndex(0); operation->MutableTableId()->SetTableId(tableId); operation->MutableTableId()->SetSchemaVersion(schemaVersion); + operation->SetIsBulk(true); } }; diff --git a/ydb/core/tx/data_events/write_data.h b/ydb/core/tx/data_events/write_data.h index 332f395e31c4..f143bf11ba09 100644 --- a/ydb/core/tx/data_events/write_data.h +++ b/ydb/core/tx/data_events/write_data.h @@ -53,6 +53,8 @@ class TWriteMeta: public NColumnShard::TMonitoringObjectsCounter, TN const std::shared_ptr Counters; mutable NOlap::NCounters::TStateSignalsOperator::TGuard StateGuard; + YDB_FLAG_ACCESSOR(Bulk, false); + public: void OnStage(const EWriteStage stage) const; From 10b069feca8dfcc59bdcf064069db23420dcfe86 Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Tue, 1 Jul 2025 10:49:07 +0200 Subject: [PATCH 48/62] wait time has been added (#16415) --- ydb/core/tx/columnshard/engines/reader/actor/actor.cpp | 5 +++++ ydb/core/tx/columnshard/engines/reader/actor/actor.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index bce7f1bb383f..e33083072bc6 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -75,6 +75,9 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { } void TColumnShardScan::HandleScan(NColumnShard::TEvPrivate::TEvTaskProcessedResult::TPtr& ev) { + if (ChunksLimiter.HasMore()) { + WaitTime = TInstant::Now() - StartWaitTime; + } auto g = Stats->MakeGuard("task_result"); auto result = ev->Get()->ExtractResult(); if (result.IsFail()) { @@ -91,6 +94,7 @@ void TColumnShardScan::HandleScan(NColumnShard::TEvPrivate::TEvTaskProcessedResu } void TColumnShardScan::HandleScan(NKqp::TEvKqpCompute::TEvScanDataAck::TPtr& ev) { + StartWaitTime = TInstant::Now(); auto g = Stats->MakeGuard("ack"); AFL_VERIFY(!AckReceivedInstant); @@ -388,6 +392,7 @@ bool TColumnShardScan::SendResult(bool pageFault, bool lastBatch) { LastResultInstant = TMonotonic::Now(); Result->CpuTime = ScanCountersPool.GetExecutionDuration(); + Result->WaitTime = WaitTime; Send(ScanComputeActorId, Result.Release(), IEventHandle::FlagTrackDelivery); // TODO: FlagSubscribeOnSession ? diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.h b/ydb/core/tx/columnshard/engines/reader/actor/actor.h index f50dee364195..600db063e3a8 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.h +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.h @@ -193,6 +193,8 @@ class TColumnShardScan: public TActorBootstrapped, ui64 PacksSum = 0; ui64 Bytes = 0; ui32 PageFaults = 0; + TInstant StartWaitTime; + TDuration WaitTime; }; } // namespace NKikimr::NOlap::NReader From 01b2ed5b704da8dcdaf4a12f0a286b05beab9242 Mon Sep 17 00:00:00 2001 From: Hor911 Date: Mon, 30 Jun 2025 13:32:16 +0300 Subject: [PATCH 49/62] Correct Scan Generation in case of retries (#20191) (#20347) --- .../kqp_scan_compute_manager.cpp | 1 + .../compute_actor/kqp_scan_compute_manager.h | 3 +- ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp | 76 +++++++++++++++++++ ydb/core/kqp/runtime/ut/ya.make | 4 + 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp index 1b408cdf6f82..a727dac65cfd 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp @@ -9,6 +9,7 @@ TShardState::TPtr TInFlightShards::Put(TShardState&& state) { MutableStatistics(state.TabletId).MutableStatistics(0).SetStartInstant(Now()); TShardState::TPtr result = std::make_shared(std::move(state)); + result->Generation = 1; AFL_ENSURE(Shards.emplace(result->TabletId, result).second)("tablet_id", result->TabletId); return result; } diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h index 5114ab963299..1f00ec9a169f 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h @@ -57,7 +57,7 @@ class TShardScannerInfo { TShardScannerInfo(const ui64 scanId, TShardState& state, const IExternalObjectsProvider& externalObjectsProvider) : ScanId(scanId) , TabletId(state.TabletId) - , Generation(++state.Generation) + , Generation(state.Generation) { const bool subscribed = std::exchange(state.SubscribedOnTablet, true); @@ -368,6 +368,7 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { state->ActorId = {}; state->State = NComputeActor::EShardState::Initial; state->SubscribedOnTablet = false; + state->Generation++; auto it = ShardScanners.find(tabletId); AFL_ENSURE(it != ShardScanners.end())("tablet_id", tabletId); diff --git a/ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp b/ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp new file mode 100644 index 000000000000..0a19d4c9424a --- /dev/null +++ b/ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp @@ -0,0 +1,76 @@ +#include + +#include +#include + +#include + +Y_UNIT_TEST_SUITE(TKqpScanFetcher) { + + Y_UNIT_TEST(ScanDelayedRetry) { + + constexpr ui64 TABLET_ID = 1001001; + + NActors::TTestActorRuntime runtime; + runtime.Initialize(NKikimr::TAppPrepare().Unwrap()); + + auto pipeCache = runtime.AllocateEdgeActor(); + runtime.RegisterService(NKikimr::MakePipePerNodeCacheID(false), pipeCache); + auto scan = runtime.AllocateEdgeActor(); + auto compute = runtime.AllocateEdgeActor(); + + NKikimrKqp::TKqpSnapshot snapshot; + NYql::NDq::TComputeRuntimeSettings settings; + NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta meta; + auto& read = *meta.AddReads(); + read.SetShardId(TABLET_ID); + read.AddKeyRanges(); + NKikimrConfig::TTableServiceConfig::TShardsScanningPolicy protoPolicy; + NKikimr::NKqp::TShardsScanningPolicy shardsScanningPolicy(protoPolicy); + NWilson::TTraceId traceId(0); + NKikimr::NKqp::TCPULimits cpuLimits; + NMonitoring::TDynamicCounterPtr counters = MakeIntrusive(); + auto scanFetcher = runtime.Register(CreateKqpScanFetcher(snapshot, {compute}, meta, settings, + 0, TMaybe(), 0, TMaybe(), shardsScanningPolicy, + MakeIntrusive(counters), 0, cpuLimits) + ); + runtime.EnableScheduleForActor(scanFetcher, true); + + // 1. Simulate fail + { + auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); + NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); + UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 1); + } + runtime.Send(scanFetcher, pipeCache, new NKikimr::TEvPipeCache::TEvDeliveryProblem(TABLET_ID, false)); + + // 2. First fail is retried instantly, so fail again + { + auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); + NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); + UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 2); + } + runtime.Send(scanFetcher, pipeCache, new NKikimr::TEvPipeCache::TEvDeliveryProblem(TABLET_ID, false)); + + // 3. Now we have 250ms until next retry, send late reply + runtime.Send(scanFetcher, scan, new NKikimr::NKqp::TEvKqpCompute::TEvScanInitActor(0, scan, 2, TABLET_ID, true)); + + // 4. Check for Fetcher failure + { + auto event = runtime.GrabEdgeEvent(TSet{compute}, TDuration::Seconds(1)); + UNIT_ASSERT_C(!event, "Unexpected TEvTerminateFromFetcher"); + } + + // 5. Yet another retry + { + auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); + NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); + UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 3); + } + runtime.Send(scanFetcher, scan, new NKikimr::NKqp::TEvKqpCompute::TEvScanInitActor(0, scan, 3, TABLET_ID, true)); + { + auto event = runtime.GrabEdgeEvent(TSet{scan}); + Y_UNUSED(event); + } + } +} diff --git a/ydb/core/kqp/runtime/ut/ya.make b/ydb/core/kqp/runtime/ut/ya.make index a3c2ea159257..3bdeed7c71d7 100644 --- a/ydb/core/kqp/runtime/ut/ya.make +++ b/ydb/core/kqp/runtime/ut/ya.make @@ -7,6 +7,7 @@ SIZE(MEDIUM) SRCS( kqp_scan_data_ut.cpp kqp_compute_scheduler_ut.cpp + kqp_scan_fetcher_ut.cpp ) YQL_LAST_ABI_VERSION() @@ -16,6 +17,9 @@ PEERDIR( ydb/core/testlib/basics/pg yql/essentials/minikql/comp_nodes/llvm16 yql/essentials/public/udf/service/exception_policy + yt/yql/providers/yt/codec/codegen + yt/yql/providers/yt/comp_nodes/llvm16 + yt/yql/providers/yt/comp_nodes/dq/llvm16 ) END() From 4669b0bff2158ff857b5285f2aa743f61b5f61a7 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Tue, 1 Jul 2025 19:38:17 +0300 Subject: [PATCH 50/62] tests for shards resolving (#20419) Co-authored-by: ivanmorozov333 --- ydb/core/kqp/common/ya.make | 1 + .../kqp/compute_actor/kqp_compute_actor.cpp | 10 +- .../kqp/compute_actor/kqp_compute_actor.h | 13 +- .../kqp/compute_actor/kqp_compute_state.h | 2 +- .../kqp_scan_compute_manager.cpp | 2 +- .../compute_actor/kqp_scan_compute_manager.h | 83 ++++--- .../compute_actor/kqp_scan_fetcher_actor.cpp | 44 +--- .../compute_actor/kqp_scan_fetcher_actor.h | 41 ++-- ydb/core/kqp/runtime/kqp_scan_fetcher_ut.cpp | 10 +- ydb/core/kqp/ut/olap/kqp_olap_ut.cpp | 212 ++++++++++++------ ydb/core/protos/table_service_config.proto | 2 + ydb/core/testlib/test_client.h | 4 + .../tx/columnshard/hooks/abstract/abstract.h | 46 ++++ ydb/library/accessor/positive_integer.cpp | 2 +- 14 files changed, 294 insertions(+), 178 deletions(-) diff --git a/ydb/core/kqp/common/ya.make b/ydb/core/kqp/common/ya.make index 938dc883ee02..5d7f6554e37e 100644 --- a/ydb/core/kqp/common/ya.make +++ b/ydb/core/kqp/common/ya.make @@ -55,6 +55,7 @@ YQL_LAST_ABI_VERSION() GENERATE_ENUM_SERIALIZATION(kqp_tx_info.h) GENERATE_ENUM_SERIALIZATION(kqp_yql.h) +GENERATE_ENUM_SERIALIZATION(kqp_resolve.h) END() diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp index 9c27bde4b5c5..3098209d5705 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.cpp @@ -159,11 +159,11 @@ IActor* CreateKqpScanComputeActor(const TActorId& executerId, ui64 txId, } IActor* CreateKqpScanFetcher(const NKikimrKqp::TKqpSnapshot& snapshot, std::vector&& computeActors, - const NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta& meta, const NYql::NDq::TComputeRuntimeSettings& settings, - const ui64 txId, TMaybe lockTxId, ui32 lockNodeId, TMaybe lockMode, - const TShardsScanningPolicy& shardsScanningPolicy, TIntrusivePtr counters, NWilson::TTraceId traceId, - const TCPULimits& cpuLimits) { - return new NScanPrivate::TKqpScanFetcherActor(snapshot, settings, std::move(computeActors), txId, lockTxId, lockNodeId, lockMode, meta, shardsScanningPolicy, counters, std::move(traceId), cpuLimits); + const NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta& meta, const NYql::NDq::TComputeRuntimeSettings& settings, const ui64 txId, + TMaybe lockTxId, ui32 lockNodeId, TMaybe lockMode, const TShardsScanningPolicy& shardsScanningPolicy, + TIntrusivePtr counters, NWilson::TTraceId traceId, const TCPULimits& cpuLimits) { + return new NScanPrivate::TKqpScanFetcherActor(snapshot, settings, std::move(computeActors), txId, lockTxId, lockNodeId, lockMode, meta, + shardsScanningPolicy, counters, std::move(traceId), cpuLimits); } } diff --git a/ydb/core/kqp/compute_actor/kqp_compute_actor.h b/ydb/core/kqp/compute_actor/kqp_compute_actor.h index e1ddc99327d7..bb45c0365fc4 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_actor.h @@ -31,6 +31,14 @@ class TShardsScanningPolicy { } + ui32 GetCriticalTotalRetriesCount() const { + return ProtoConfig.GetCriticalTotalRetriesCount(); + } + + ui32 GetReaskShardRetriesCount() const { + return ProtoConfig.GetReaskShardRetriesCount(); + } + bool IsParallelScanningAvailable() const { return ProtoConfig.GetParallelScanningAvailable(); } @@ -68,9 +76,8 @@ IActor* CreateKqpScanComputeActor(const TActorId& executerId, ui64 txId, NYql::N IActor* CreateKqpScanFetcher(const NKikimrKqp::TKqpSnapshot& snapshot, std::vector&& computeActors, const NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta& meta, const NYql::NDq::TComputeRuntimeSettings& settings, - const ui64 txId, TMaybe lockTxId, ui32 lockNodeId, TMaybe lockMode, - const TShardsScanningPolicy& shardsScanningPolicy, TIntrusivePtr counters, NWilson::TTraceId traceId, - const TCPULimits& cpuLimits); + const ui64 txId, TMaybe lockTxId, ui32 lockNodeId, TMaybe lockMode, const TShardsScanningPolicy& shardsScanningPolicy, + TIntrusivePtr counters, NWilson::TTraceId traceId, const TCPULimits& cpuLimits); NYql::NDq::IDqAsyncIoFactory::TPtr CreateKqpAsyncIoFactory( TIntrusivePtr counters, diff --git a/ydb/core/kqp/compute_actor/kqp_compute_state.h b/ydb/core/kqp/compute_actor/kqp_compute_state.h index e282d78e5cab..ce1fe0e081bf 100644 --- a/ydb/core/kqp/compute_actor/kqp_compute_state.h +++ b/ydb/core/kqp/compute_actor/kqp_compute_state.h @@ -34,7 +34,7 @@ struct TShardState: public TCommonRetriesState { const ui64 TabletId; TSmallVec Ranges; EShardState State = EShardState::Initial; - ui32 Generation = 0; + ui32 Generation = 1; bool SubscribedOnTablet = false; TActorId ActorId; TOwnedCellVec LastKey; diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp index a727dac65cfd..2b54770f9646 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.cpp @@ -5,11 +5,11 @@ namespace NKikimr::NKqp::NScanPrivate { TShardState::TPtr TInFlightShards::Put(TShardState&& state) { + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "put_inflight")("tablet_id", state.TabletId)("state", state.State)("gen", state.Generation); TScanShardsStatistics::OnScansDiff(Shards.size(), GetScansCount()); MutableStatistics(state.TabletId).MutableStatistics(0).SetStartInstant(Now()); TShardState::TPtr result = std::make_shared(std::move(state)); - result->Generation = 1; AFL_ENSURE(Shards.emplace(result->TabletId, result).second)("tablet_id", result->TabletId); return result; } diff --git a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h index 1f00ec9a169f..3ed11950eae0 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_compute_manager.h @@ -1,21 +1,24 @@ #pragma once -#include "kqp_scan_common.h" #include "kqp_compute_actor.h" #include "kqp_compute_state.h" +#include "kqp_scan_common.h" #include "kqp_scan_compute_stat.h" +#include "kqp_scan_events.h" + #include -#include +#include +#include #include #include -#include -#include "kqp_scan_events.h" + +#include namespace NKikimr::NKqp::NScanPrivate { class IExternalObjectsProvider { public: - virtual std::unique_ptr BuildEvKqpScan(const ui32 scanId, const ui32 gen, const TSmallVec& ranges, - const std::optional& cursor) const = 0; + virtual std::unique_ptr BuildEvKqpScan(const ui32 scanId, const ui32 gen, + const TSmallVec& ranges, const std::optional& cursor) const = 0; virtual const TVector& GetKeyColumnTypes() const = 0; }; @@ -50,27 +53,28 @@ class TShardScannerInfo { TracingStarted = true; } if (NActors::TlsActivationContext) { - NActors::TActivationContext::AsActorContext().Send(*ActorId, new TEvKqpCompute::TEvScanDataAck(FreeSpace, Generation, 1), flags, TabletId); + NActors::TActivationContext::AsActorContext().Send( + *ActorId, new TEvKqpCompute::TEvScanDataAck(FreeSpace, Generation, 1), flags, TabletId); } } + public: TShardScannerInfo(const ui64 scanId, TShardState& state, const IExternalObjectsProvider& externalObjectsProvider) : ScanId(scanId) , TabletId(state.TabletId) - , Generation(state.Generation) - { + , Generation(state.Generation) { const bool subscribed = std::exchange(state.SubscribedOnTablet, true); const auto& keyColumnTypes = externalObjectsProvider.GetKeyColumnTypes(); auto ranges = state.GetScanRanges(keyColumnTypes); auto ev = externalObjectsProvider.BuildEvKqpScan(ScanId, Generation, ranges, state.LastCursorProto); - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner")("tablet_id", TabletId)("generation", Generation) - ("info", state.ToString(keyColumnTypes))("range", DebugPrintRanges(keyColumnTypes, ranges, *AppData()->TypeRegistry))( + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner")("tablet_id", TabletId)("generation", Generation)( + "info", state.ToString(keyColumnTypes))("range", DebugPrintRanges(keyColumnTypes, ranges, *AppData()->TypeRegistry))( "subscribed", subscribed)("cursor", state.CursorDebugString()); - NActors::TActivationContext::AsActorContext().Send(MakePipePerNodeCacheID(false), - new TEvPipeCache::TEvForward(ev.release(), TabletId, !subscribed), IEventHandle::FlagTrackDelivery); + NActors::TActivationContext::AsActorContext().Send( + MakePipePerNodeCacheID(false), new TEvPipeCache::TEvForward(ev.release(), TabletId, !subscribed), IEventHandle::FlagTrackDelivery); } ui64 GetTabletId() const { @@ -92,11 +96,13 @@ class TShardScannerInfo { void Stop(const bool finalFlag, const TString& message) { AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "stop_scanner")("actor_id", ActorId)("message", message)("final_flag", finalFlag); if (ActorId) { - auto abortEv = std::make_unique(NYql::NDqProto::StatusIds::CANCELLED, message ? message : "stop from fetcher"); + auto abortEv = + std::make_unique(NYql::NDqProto::StatusIds::CANCELLED, message ? message : "stop from fetcher"); NActors::TActivationContext::AsActorContext().Send(*ActorId, std::move(abortEv)); if (finalFlag) { NActors::TActivationContext::AsActorContext().Send(MakePipePerNodeCacheID(false), new TEvPipeCache::TEvUnlink(TabletId)); - NActors::TActivationContext::AsActorContext().Send(TActivationContext::InterconnectProxy(ActorId->NodeId()), new TEvents::TEvUnsubscribe()); + NActors::TActivationContext::AsActorContext().Send( + TActivationContext::InterconnectProxy(ActorId->NodeId()), new TEvents::TEvUnsubscribe()); } ActorId = {}; } @@ -120,7 +126,8 @@ class TShardScannerInfo { DoAck(); } - std::vector> OnReceiveData(TEvKqpCompute::TEvScanData& data, const std::shared_ptr& selfPtr); + std::vector> OnReceiveData( + TEvKqpCompute::TEvScanData& data, const std::shared_ptr& selfPtr); void FinishWaitSendData() { --DataChunksInFlightCount; @@ -132,11 +139,12 @@ class TShardScannerInfo { }; class TComputeTaskData { -private: +public: std::shared_ptr Info; std::unique_ptr Event; bool Finished = false; const std::optional ComputeShardId; + public: ui32 GetRowsCount() const { AFL_ENSURE(Event); @@ -151,11 +159,11 @@ class TComputeTaskData { return ComputeShardId; } - TComputeTaskData(const std::shared_ptr& info, std::unique_ptr&& ev, const std::optional computeShardId = std::optional()) + TComputeTaskData(const std::shared_ptr& info, std::unique_ptr&& ev, + const std::optional computeShardId = std::optional()) : Info(info) , Event(std::move(ev)) - , ComputeShardId(computeShardId) - { + , ComputeShardId(computeShardId) { } void Finish() { @@ -175,8 +183,8 @@ class TInFlightComputes { std::deque> DataQueue; bool SendData() { - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "send_data_to_compute")("space", FreeSpace)("queue", DataQueue.size())("compute_actor_id", ActorId) - ("rows", DataQueue.size() ? DataQueue.front()->GetRowsCount() : 0); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "send_data_to_compute")("space", FreeSpace)("queue", DataQueue.size())( + "compute_actor_id", ActorId)("rows", DataQueue.size() ? DataQueue.front()->GetRowsCount() : 0); if (FreeSpace && DataQueue.size()) { NActors::TActivationContext::AsActorContext().Send(ActorId, DataQueue.front()->ExtractEvent()); DataQueue.front()->Finish(); @@ -194,7 +202,6 @@ class TInFlightComputes { TComputeActorInfo(const NActors::TActorId& actorId) : ActorId(actorId) { - } void OnAckReceived(const ui64 freeSpace) { @@ -219,6 +226,7 @@ class TInFlightComputes { std::deque> UndefinedShardTaskData; std::deque ComputeActors; THashMap ComputeActorsById; + public: ui32 GetPacksToSendCount() const { ui32 result = UndefinedShardTaskData.size(); @@ -237,8 +245,7 @@ class TInFlightComputes { TString DebugString() const { TStringBuilder sb; - sb << "ca=" << ComputeActors.size() << ";" - ; + sb << "ca=" << ComputeActors.size() << ";"; return sb; } @@ -268,7 +275,7 @@ class TInFlightComputes { } UndefinedShardTaskData.emplace_back(std::move(sendTask)); } else { - AFL_ENSURE(*computeShardId < ComputeActors.size())("compute_shard_id", *computeShardId); + AFL_ENSURE (*computeShardId < ComputeActors.size())("compute_shard_id", *computeShardId); ComputeActors[*computeShardId].AddDataToSend(std::move(sendTask)); } } @@ -283,8 +290,8 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { THashMap> ShardScanners; const ui64 ScanId; const IExternalObjectsProvider& ExternalObjectsProvider; -public: +public: void AbortAllScanners(const TString& errorMessage) { AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "abort_all_scanners")("error_message", errorMessage); for (auto&& itTablet : ShardScanners) { @@ -324,15 +331,15 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { void RegisterScannerActor(const ui64 tabletId, const ui64 generation, const TActorId& scanActorId, bool allowPings) { auto state = GetShardState(tabletId); if (!state || generation != state->Generation) { - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "register_scanner_actor_dropped") - ("actor_id", scanActorId)("is_state_initialized", !!state)("generation", generation); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "register_scanner_actor_dropped")("actor_id", scanActorId)( + "is_state_initialized", !!state)("generation", generation); return; } - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "register_scanner_actor")("actor_id", scanActorId) - ("state", state->State)("tablet_id", state->TabletId)("generation", state->Generation); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "register_scanner_actor")("actor_id", scanActorId)("state", state->State)( + "tablet_id", state->TabletId)("generation", state->Generation)("shard_actor", state->ActorId); - AFL_ENSURE(state->State == NComputeActor::EShardState::Starting)("state", state->State); + AFL_ENSURE(state->State == NComputeActor::EShardState::Starting)("state", state->State)("tablet_id", tabletId); AFL_ENSURE(!state->ActorId)("actor_id", state->ActorId); state->State = NComputeActor::EShardState::Running; @@ -344,8 +351,9 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { } void StartScanner(TShardState& state) { - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner") - ("state", state.State)("tablet_id", state.TabletId)("generation", state.Generation); + NYDBTest::TControllers::GetKqpController()->OnInitTabletScan(state.TabletId); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "start_scanner")("state", state.State)("tablet_id", state.TabletId)( + "generation", state.Generation); AFL_ENSURE(state.State == NComputeActor::EShardState::Initial)("state", state.State); AFL_ENSURE(state.TabletId); @@ -374,6 +382,8 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { AFL_ENSURE(it != ShardScanners.end())("tablet_id", tabletId); it->second->Stop(true, ""); ShardScanners.erase(it); + } else { + AFL_VERIFY(!state->ActorId); } if (stopShard) { @@ -397,8 +407,7 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { TInFlightShards(const ui64 scanId, const IExternalObjectsProvider& externalObjectsProvider) : ScanId(scanId) - , ExternalObjectsProvider(externalObjectsProvider) - { + , ExternalObjectsProvider(externalObjectsProvider) { } bool IsActive() const { return IsActiveFlag; @@ -418,4 +427,4 @@ class TInFlightShards: public NComputeActor::TScanShardsStatistics { TShardState::TPtr Put(TShardState&& state); }; -} +} // namespace NKikimr::NKqp::NScanPrivate diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp index 569c35b993b6..f48628ed19d2 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.cpp @@ -17,10 +17,6 @@ using namespace NYql; using namespace NYql::NDq; using namespace NKikimr::NKqp::NComputeActor; -static constexpr ui64 MAX_SHARD_RETRIES = 5; // retry after: 0, 250, 500, 1000, 2000 -static constexpr ui64 MAX_TOTAL_SHARD_RETRIES = 20; -static constexpr ui64 MAX_SHARD_RESOLVES = 3; - constexpr TDuration REGISTRATION_TIMEOUT = TDuration::Seconds(60); constexpr TDuration PING_PERIOD = TDuration::Seconds(30); @@ -45,10 +41,10 @@ TKqpScanFetcherActor::TKqpScanFetcherActor(const NKikimrKqp::TKqpSnapshot& snaps , Counters(counters) , InFlightShards(ScanId, *this) , InFlightComputes(ComputeActorIds) - , IsOlapTable(Meta.GetTable().GetTableKind() == (ui32)ETableKind::Olap) { + , TableKind(Meta.GetTable().HasTableKind() ? (NKqp::ETableKind)Meta.GetTable().GetTableKind() : NKqp::ETableKind::Unknown) { Y_UNUSED(traceId); AFL_ENSURE(!Meta.GetReads().empty()); - AFL_ENSURE(Meta.GetTable().GetTableKind() != (ui32)ETableKind::SysView); + AFL_ENSURE(TableKind != NKqp::ETableKind::SysView); ALS_DEBUG(NKikimrServices::KQP_COMPUTE) << "META:" << meta.DebugString(); KeyColumnTypes.reserve(Meta.GetKeyColumnTypes().size()); for (size_t i = 0; i < Meta.KeyColumnTypesSize(); i++) { @@ -87,8 +83,8 @@ void TKqpScanFetcherActor::Bootstrap() { for (auto&& c : ComputeActorIds) { Sender().SendTo(c); } - AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "bootstrap")("self_if", SelfId())("compute", ComputeActorIds.size())( - "shards", PendingShards.size()); + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "bootstrap")("compute", ComputeActorIds.size())("shards", PendingShards.size())( + "self_id", SelfId()); StartTableScan(); Become(&TKqpScanFetcherActor::StateFunc); Schedule(PING_PERIOD, new NActors::TEvents::TEvWakeup()); @@ -184,14 +180,6 @@ void TKqpScanFetcherActor::HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) } if (state->State == EShardState::Starting) { - ++TotalRetries; - if (TotalRetries >= MAX_TOTAL_SHARD_RETRIES) { - CA_LOG_E("TKqpScanFetcherActor: broken tablet for this request " << state->TabletId << ", retries limit exceeded (" - << state->TotalRetries << "/" << TotalRetries << ")"); - SendGlobalFail(NDqProto::COMPUTE_STATE_FAILURE, YdbStatusToDqStatus(status), issues); - return PassAway(); - } - if (FindSchemeErrorInIssues(status, issues)) { return EnqueueResolveShard(state); } @@ -208,7 +196,7 @@ void TKqpScanFetcherActor::HandleExecute(TEvKqpCompute::TEvScanError::TPtr& ev) if (state->State == EShardState::PostRunning || state->State == EShardState::Running) { CA_LOG_E("TKqpScanFetcherActor: broken tablet for this request " << state->TabletId << ", retries limit exceeded (" - << state->TotalRetries << "/" << TotalRetries << ")"); + << state->TotalRetries << ")"); SendGlobalFail(NDqProto::COMPUTE_STATE_FAILURE, YdbStatusToDqStatus(status), issues); return PassAway(); } @@ -303,7 +291,9 @@ void TKqpScanFetcherActor::HandleExecute(TEvTxProxySchemeCache::TEvResolveKeySet } const auto& tr = *AppData()->TypeRegistry; - if (IsOlapTable) { + AFL_DEBUG(NKikimrServices::KQP_COMPUTE)("event", "on_resolving")("tablet_id", state.TabletId)("state", state.State)("gen", state.Generation)( + "kind", TableKind)("has_program", Meta.HasOlapProgram()); + if (TableKind == NKqp::ETableKind::Olap) { bool found = false; for (auto&& partition : keyDesc->GetPartitions()) { if (partition.ShardId != state.TabletId) { @@ -311,6 +301,7 @@ void TKqpScanFetcherActor::HandleExecute(TEvTxProxySchemeCache::TEvResolveKeySet } auto newShard = TShardState(partition.ShardId); AFL_ENSURE(!found); + newShard.Generation = state.Generation; newShard.LastKey = std::move(state.LastKey); newShard.LastCursorProto = std::move(state.LastCursorProto); newShard.Ranges = state.Ranges; @@ -594,7 +585,6 @@ void TKqpScanFetcherActor::StartTableScan() { if (!isFirst) { CA_LOG_D("AFTER: " << PendingShards.size() << "." << InFlightShards.GetScansCount() << "." << PendingResolveShards.size()); } - CA_LOG_D("Scheduled table scans, in flight: " << InFlightShards.GetScansCount() << " shards. " << "pending shards to read: " << PendingShards.size() << ", " << "pending resolve shards: " << PendingResolveShards.size() << ", " @@ -606,7 +596,7 @@ void TKqpScanFetcherActor::RetryDeliveryProblem(TShardState::TPtr state) { InFlightShards.StopScanner(state->TabletId, false); Counters->ScanQueryShardDisconnect->Inc(); - if (state->TotalRetries >= MAX_TOTAL_SHARD_RETRIES) { + if (state->TotalRetries >= ShardsScanningPolicy.GetCriticalTotalRetriesCount()) { CA_LOG_E( "TKqpScanFetcherActor: broken pipe with tablet " << state->TabletId << ", retries limit exceeded (" << state->TotalRetries << ")"); SendGlobalFail(NDqProto::StatusIds::UNAVAILABLE, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, @@ -614,17 +604,12 @@ void TKqpScanFetcherActor::RetryDeliveryProblem(TShardState::TPtr state) { return; } - // note: it might be possible that shard is already removed after successful split/merge operation and cannot be found - // in this case the next TEvKqpScan request will receive the delivery problem response. - // so after several consecutive delivery problem responses retry logic should - // resolve shard details again. - if (state->RetryAttempt >= MAX_SHARD_RETRIES) { + if (state->RetryAttempt >= ShardsScanningPolicy.GetReaskShardRetriesCount()) { state->ResetRetry(); Send(state->ActorId, new NActors::TEvents::TEvPoisonPill()); return EnqueueResolveShard(state); } - ++TotalRetries; auto retryDelay = state->CalcRetryDelay(); CA_LOG_W("TKqpScanFetcherActor: broken pipe with tablet " << state->TabletId << ", restarting scan from last received key " << state->PrintLastKey(KeyColumnTypes) << ", attempt #" << state->RetryAttempt @@ -636,12 +621,6 @@ void TKqpScanFetcherActor::RetryDeliveryProblem(TShardState::TPtr state) { } void TKqpScanFetcherActor::ResolveShard(TShardState& state) { - if (state.ResolveAttempt >= MAX_SHARD_RESOLVES) { - SendGlobalFail(NDqProto::StatusIds::UNAVAILABLE, TIssuesIds::KIKIMR_TEMPORARILY_UNAVAILABLE, - TStringBuilder() << "Table '" << ScanDataMeta.TablePath << "' resolve limit exceeded"); - return; - } - Counters->ScanQueryShardResolve->Inc(); state.State = EShardState::Resolving; @@ -676,6 +655,7 @@ void TKqpScanFetcherActor::ResolveShard(TShardState& state) { void TKqpScanFetcherActor::EnqueueResolveShard(const std::shared_ptr& state) { CA_LOG_D("Enqueue for resolve " << state->TabletId); InFlightShards.StopScanner(state->TabletId); + NYDBTest::TControllers::GetKqpController()->OnInitTabletResolving(state->TabletId); PendingResolveShards.emplace_back(*state); if (PendingResolveShards.size() == 1) { ResolveNextShard(); diff --git a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h index 5ef0a9d82c10..38b3b207e9dd 100644 --- a/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h +++ b/ydb/core/kqp/compute_actor/kqp_scan_fetcher_actor.h @@ -1,25 +1,25 @@ #pragma once -#include "kqp_compute_events.h" #include "kqp_compute_actor.h" -#include "kqp_scan_events.h" +#include "kqp_compute_events.h" #include "kqp_compute_state.h" -#include "kqp_scan_compute_manager.h" #include "kqp_scan_common.h" +#include "kqp_scan_compute_manager.h" +#include "kqp_scan_events.h" #include - +#include +#include #include -#include #include -#include -#include +#include #include -#include +#include #include #include -#include #include +#include +#include namespace NKikimr::NKqp::NScanPrivate { @@ -36,6 +36,7 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped&& computeActors, - const ui64 txId, const TMaybe lockTxId, const ui32 lockNodeId, const TMaybe lockMode, - const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, + std::vector&& computeActors, const ui64 txId, const TMaybe lockTxId, const ui32 lockNodeId, + const TMaybe lockMode, const NKikimrTxDataShard::TKqpTransaction_TScanTaskMeta& meta, const TShardsScanningPolicy& shardsScanningPolicy, TIntrusivePtr counters, NWilson::TTraceId traceId, const TCPULimits& cpuLimits); - static TVector BuildSerializedTableRanges(const NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta::TReadOpMeta& readData); + static TVector BuildSerializedTableRanges( + const NKikimrTxDataShard::TKqpTransaction::TScanTaskMeta::TReadOpMeta& readData); void Bootstrap(); @@ -105,16 +106,17 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped ComputeActorIds; void StopOnError(const TString& errorMessage) const; - bool SendGlobalFail(const NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssuesIds::EIssueCode issueCode, const TString& message) const; + bool SendGlobalFail( + const NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssuesIds::EIssueCode issueCode, const TString& message) const; - bool SendGlobalFail(const NYql::NDqProto::EComputeState state, NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssues& issues) const; + bool SendGlobalFail( + const NYql::NDqProto::EComputeState state, NYql::NDqProto::StatusIds::StatusCode statusCode, const NYql::TIssues& issues) const; bool SendScanFinished(); @@ -124,7 +126,6 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped TrackingNodes; ui32 MaxInFlight = 1024; @@ -196,4 +195,4 @@ class TKqpScanFetcherActor: public NActors::TActorBootstrapped #include +#include Y_UNIT_TEST_SUITE(TKqpScanFetcher) { @@ -30,17 +31,18 @@ Y_UNIT_TEST_SUITE(TKqpScanFetcher) { NWilson::TTraceId traceId(0); NKikimr::NKqp::TCPULimits cpuLimits; NMonitoring::TDynamicCounterPtr counters = MakeIntrusive(); - auto scanFetcher = runtime.Register(CreateKqpScanFetcher(snapshot, {compute}, meta, settings, + auto scanFetcher = runtime.Register(CreateKqpScanFetcher(snapshot, { compute }, meta, settings, 0, TMaybe(), 0, TMaybe(), shardsScanningPolicy, MakeIntrusive(counters), 0, cpuLimits) ); runtime.EnableScheduleForActor(scanFetcher, true); + NKikimr::TPositiveIncreasingControlInteger controlGeneration; // 1. Simulate fail { auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); - UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 1); + controlGeneration = NKikimr::TPositiveIncreasingControlInteger(evScan->Record.GetGeneration()); } runtime.Send(scanFetcher, pipeCache, new NKikimr::TEvPipeCache::TEvDeliveryProblem(TABLET_ID, false)); @@ -48,7 +50,7 @@ Y_UNIT_TEST_SUITE(TKqpScanFetcher) { { auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); - UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 2); + controlGeneration = NKikimr::TPositiveIncreasingControlInteger(evScan->Record.GetGeneration()); } runtime.Send(scanFetcher, pipeCache, new NKikimr::TEvPipeCache::TEvDeliveryProblem(TABLET_ID, false)); @@ -65,7 +67,7 @@ Y_UNIT_TEST_SUITE(TKqpScanFetcher) { { auto event = runtime.GrabEdgeEvent(TSet{pipeCache}); NKikimr::TEvDataShard::TEvKqpScan* evScan = dynamic_cast(event->Get()->Ev.get()); - UNIT_ASSERT_EQUAL(evScan->Record.GetGeneration(), 3); + controlGeneration = NKikimr::TPositiveIncreasingControlInteger(evScan->Record.GetGeneration()); } runtime.Send(scanFetcher, scan, new NKikimr::NKqp::TEvKqpCompute::TEvScanInitActor(0, scan, 3, TABLET_ID, true)); { diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index aa4fc7a8acc6..bcdc56e38098 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1937,100 +1937,166 @@ Y_UNIT_TEST_SUITE(KqpOlap) { UNIT_ASSERT(hasResult); } - Y_UNIT_TEST(ManyColumnShardsWithRestarts) { - TPortManager tp; - ui16 mbusport = tp.GetPort(2134); - auto settings = Tests::TServerSettings(mbusport) - .SetDomainName("Root") - .SetUseRealThreads(false).SetNodeCount(2).SetColumnShardReaderClassName( - "SIMPLE"); - - Tests::TServer::TPtr server = new Tests::TServer(settings); - auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); - - auto runtime = server->GetRuntime(); - auto sender = runtime->AllocateEdgeActor(); + class TManyColumnShardsWithRestartsExecutor { + private: + virtual void FillSettings(Tests::TServerSettings& settings) const = 0; + virtual void OnAfterTest() const = 0; - InitRoot(server, sender); -// Tests::NCommon::TLoggerInit(runtime).Initialize(); + Tests::TServerSettings BuildSettings() { + Tests::TServerSettings settings(PortsManager.GetPort(2134)); + FillSettings(settings); + return settings; + } - ui32 numShards = NSan::PlainOrUnderSanitizer(100, 10); - ui32 numIterations = NSan::PlainOrUnderSanitizer(100, 10); - TLocalHelper(*server).SetShardingMethod("HASH_FUNCTION_CLOUD_LOGS").CreateTestOlapTable("largeOlapTable", "largeOlapStore", numShards, numShards); - ui32 insertRows = 0; + protected: + TPortManager PortsManager; - for(ui64 i = 0; i < numIterations; ++i) { - TLocalHelper(*server).SendDataViaActorSystem("/Root/largeOlapStore/largeOlapTable", 0, 1000000 + i*1000000, 2000); - insertRows += 2000; - } + public: + void Execute() { + auto settings = BuildSettings(); - ui64 result = 0; - THashSet columnShardScans; - bool prevIsFinished = false; + Tests::TServer::TPtr server = new Tests::TServer(settings); + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + auto kqpController = NYDBTest::TControllers::RegisterKqpControllerGuard(); - auto captureEvents = [&](TAutoPtr& ev) -> auto { - switch (ev->GetTypeRewrite()) { - case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { + auto runtime = server->GetRuntime(); + auto sender = runtime->AllocateEdgeActor(); - auto* msg = ev->Get(); - for (auto& [shardId, nodeId]: msg->ShardNodes) { - Cerr << "-- nodeId: " << nodeId << Endl; - nodeId = runtime->GetNodeId(0); - } - break; - } + InitRoot(server, sender); + Tests::NCommon::TLoggerInit(runtime).Initialize(); - case NKqp::TKqpExecuterEvents::EvStreamData: { - auto& record = ev->Get()->Record; + ui32 numShards = NSan::PlainOrUnderSanitizer(100, 10); + ui32 numIterations = NSan::PlainOrUnderSanitizer(100, 10); + TLocalHelper(*server) + .SetShardingMethod("HASH_FUNCTION_CLOUD_LOGS") + .CreateTestOlapTable("largeOlapTable", "largeOlapStore", numShards, numShards); + ui32 insertRows = 0; - Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); - Cerr.Flush(); + for (ui64 i = 0; i < numIterations; ++i) { + TLocalHelper(*server).SendDataViaActorSystem("/Root/largeOlapStore/largeOlapTable", 0, 1000000 + i * 1000000, 2000); + insertRows += 2000; + } - Y_ASSERT(record.GetResultSet().rows().size() == 1); - Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); - result = record.GetResultSet().rows().at(0).items().at(0).uint64_value(); + ui64 result = 0; + THashSet columnShardScans; + bool prevIsFinished = false; + + bool needCount = true; + auto captureEvents = [&](TAutoPtr& ev) -> auto { + switch (ev->GetTypeRewrite()) { + case NKqp::TKqpExecuterEvents::EvShardsResolveStatus: { + auto* msg = ev->Get(); + for (auto& [shardId, nodeId] : msg->ShardNodes) { + Cerr << "-- nodeId: " << nodeId << Endl; + nodeId = runtime->GetNodeId(0); + } + break; + } - auto resp = MakeHolder(record.GetSeqNo(), record.GetChannelId()); - resp->Record.SetEnough(false); - resp->Record.SetFreeSpace(100); - runtime->Send(new IEventHandle(ev->Sender, sender, resp.Release())); - return TTestActorRuntime::EEventAction::DROP; - } + case NKqp::TKqpExecuterEvents::EvStreamData: { + auto& record = ev->Get()->Record; - case NKqp::TKqpComputeEvents::EvScanData: { - auto [it, success] = columnShardScans.emplace(ev->Sender); - auto* msg = ev->Get(); - if (success) { - // first scan response. - prevIsFinished = msg->Finished; - return TTestActorRuntime::EEventAction::PROCESS; - } else { - if (prevIsFinished) { - Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event" << Endl); + if (needCount) { + Cerr << (TStringBuilder() << "-- EvStreamData: " << record.AsJSON() << Endl); Cerr.Flush(); - for (auto&& i : csController->GetShardActualIds()) { - runtime->Send(MakePipePerNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward( - new TEvents::TEvPoisonPill(), i, false)); - } + Y_ASSERT(record.GetResultSet().rows().size() == 1); + Y_ASSERT(record.GetResultSet().rows().at(0).items().size() == 1); + result += record.GetResultSet().rows().at(0).items().at(0).uint64_value(); } else { + Cerr << (TStringBuilder() << "-- EvStreamData: " << record.GetResultSet().rows().size() << Endl); + Cerr.Flush(); + result += record.GetResultSet().rows().size(); + } + + auto resp = MakeHolder(record.GetSeqNo(), record.GetChannelId()); + resp->Record.SetEnough(false); + resp->Record.SetFreeSpace(100); + runtime->Send(new IEventHandle(ev->Sender, sender, resp.Release())); + return TTestActorRuntime::EEventAction::DROP; + } + + case NKqp::TKqpComputeEvents::EvScanData: { + auto [it, success] = columnShardScans.emplace(ev->Sender); + auto* msg = ev->Get(); + if (success) { + // first scan response. prevIsFinished = msg->Finished; + return TTestActorRuntime::EEventAction::PROCESS; + } else { + if (prevIsFinished) { + Cerr << (TStringBuilder() << "-- EvScanData from " << ev->Sender << ": hijack event" << Endl); + Cerr.Flush(); + for (auto&& i : csController->GetShardActualIds()) { + runtime->Send(MakePipePerNodeCacheID(false), NActors::TActorId(), + new TEvPipeCache::TEvForward(new TEvents::TEvPoisonPill(), i, false)); + } + } else { + prevIsFinished = msg->Finished; + } + return TTestActorRuntime::EEventAction::PROCESS; } - return TTestActorRuntime::EEventAction::PROCESS; + break; } - break; + + default: + break; } + return TTestActorRuntime::EEventAction::PROCESS; + }; - default: - break; + runtime->SetObserverFunc(captureEvents); + { + result = 0; + auto streamSender = runtime->AllocateEdgeActor(); + NDataShard::NKqpHelpers::SendRequest(*runtime, streamSender, + NDataShard::NKqpHelpers::MakeStreamRequest( + streamSender, "SELECT COUNT(*) FROM `/Root/largeOlapStore/largeOlapTable`;", false)); + auto ev = runtime->GrabEdgeEventRethrow(streamSender); + UNIT_ASSERT_VALUES_EQUAL(result, insertRows); + } + { + result = 0; + needCount = false; + auto streamSender = runtime->AllocateEdgeActor(); + NDataShard::NKqpHelpers::SendRequest(*runtime, streamSender, + NDataShard::NKqpHelpers::MakeStreamRequest(streamSender, "SELECT * FROM `/Root/largeOlapStore/largeOlapTable`;", false)); + auto ev = runtime->GrabEdgeEventRethrow(streamSender); + UNIT_ASSERT_VALUES_EQUAL(result, insertRows); + } + OnAfterTest(); + } + }; + + Y_UNIT_TEST(ManyColumnShardsWithRestartsSimple) { + class TTestExecutor: public TManyColumnShardsWithRestartsExecutor { + private: + virtual void OnAfterTest() const override { + AFL_VERIFY(NYDBTest::TControllers::GetKqpControllerAs()->GetInitScanCounter().Val()); + AFL_VERIFY(!NYDBTest::TControllers::GetKqpControllerAs()->GetResolvingCounter().Val()); + } + virtual void FillSettings(Tests::TServerSettings& settings) const override { + settings.SetDomainName("Root").SetUseRealThreads(false).SetNodeCount(2).SetColumnShardReaderClassName("SIMPLE"); } - return TTestActorRuntime::EEventAction::PROCESS; }; + TTestExecutor().Execute(); + } - runtime->SetObserverFunc(captureEvents); - auto streamSender = runtime->AllocateEdgeActor(); - NDataShard::NKqpHelpers::SendRequest(*runtime, streamSender, NDataShard::NKqpHelpers::MakeStreamRequest(streamSender, "SELECT COUNT(*) FROM `/Root/largeOlapStore/largeOlapTable`;", false)); - auto ev = runtime->GrabEdgeEventRethrow(streamSender); - UNIT_ASSERT_VALUES_EQUAL(result, insertRows); + Y_UNIT_TEST(ManyColumnShardsWithRestartsWithResolving) { + class TTestExecutor: public TManyColumnShardsWithRestartsExecutor { + private: + virtual void OnAfterTest() const override { + AFL_VERIFY(NYDBTest::TControllers::GetKqpControllerAs()->GetInitScanCounter().Val()); + AFL_VERIFY(NYDBTest::TControllers::GetKqpControllerAs()->GetResolvingCounter().Val()); + } + virtual void FillSettings(Tests::TServerSettings& settings) const override { + settings.SetDomainName("Root") + .SetUseRealThreads(false) + .SetNodeCount(2) + .SetColumnShardReaderClassName("SIMPLE") + .SetScanReaskToResolve(1); + } + }; + TTestExecutor().Execute(); } Y_UNIT_TEST(PredicatePushdownWithParametersILike) { diff --git a/ydb/core/protos/table_service_config.proto b/ydb/core/protos/table_service_config.proto index 7b2243caaa65..ed657708d695 100644 --- a/ydb/core/protos/table_service_config.proto +++ b/ydb/core/protos/table_service_config.proto @@ -8,6 +8,8 @@ message TTableServiceConfig { optional uint32 ScanLimit = 3 [default = 16]; optional bool ParallelScanningAvailable = 4 [default = false]; optional uint32 ShardSplitFactor = 5 [default = 1]; + optional uint32 CriticalTotalRetriesCount = 6 [default = 20]; + optional uint32 ReaskShardRetriesCount = 7 [default = 5]; } message TResourceManager { diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index f2de654f6c5a..ac355a2c6a40 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -229,6 +229,10 @@ namespace Tests { TServerSettings& SetInitializeFederatedQuerySetupFactory(bool value) { InitializeFederatedQuerySetupFactory = value; return *this; } TServerSettings& SetVerbose(bool value) { Verbose = value; return *this; } TServerSettings& SetUseSectorMap(bool value) { UseSectorMap = value; return *this; } + TServerSettings& SetScanReaskToResolve(const ui32 count) { + AppConfig->MutableTableServiceConfig()->MutableResourceManager()->MutableShardsScanningPolicy()->SetReaskShardRetriesCount(count); + return *this; + } TServerSettings& SetColumnShardReaderClassName(const TString& className) { AppConfig->MutableColumnShardConfig()->SetReaderClassName(className); return *this; diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index f147f8938b0f..d6a4c55f6844 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -368,9 +368,38 @@ class ICSController { }; +class IKqpController { +public: + using TPtr = std::shared_ptr; + + virtual ~IKqpController() = default; + + virtual void OnInitTabletScan(const ui64 /*tabletId*/) { + } + + virtual void OnInitTabletResolving(const ui64 /*tabletId*/) { + } +}; + +class TTestKqpController: public IKqpController { +private: + YDB_READONLY_DEF(TAtomicCounter, InitScanCounter); + YDB_READONLY_DEF(TAtomicCounter, ResolvingCounter); + +public: + virtual void OnInitTabletScan(const ui64 /*tabletId*/) override { + InitScanCounter.Inc(); + } + + virtual void OnInitTabletResolving(const ui64 /*tabletId*/) override { + ResolvingCounter.Inc(); + } +}; + class TControllers { private: ICSController::TPtr CSController = std::make_shared(); + IKqpController::TPtr KqpController = std::make_shared(); public: template @@ -419,6 +448,23 @@ class TControllers { auto controller = Singleton()->CSController; return dynamic_cast(controller.get()); } + + template + static TGuard RegisterKqpControllerGuard(Types... args) { + auto result = std::make_shared(args...); + Singleton()->KqpController = result; + return result; + } + + static IKqpController::TPtr GetKqpController() { + return Singleton()->KqpController; + } + + template + static T* GetKqpControllerAs() { + auto controller = Singleton()->KqpController; + return dynamic_cast(controller.get()); + } }; } // namespace NKikimr::NYDBTest diff --git a/ydb/library/accessor/positive_integer.cpp b/ydb/library/accessor/positive_integer.cpp index a5aa27e75664..fe0b250175bc 100644 --- a/ydb/library/accessor/positive_integer.cpp +++ b/ydb/library/accessor/positive_integer.cpp @@ -36,7 +36,7 @@ ui64 TPositiveControlInteger::Val() const { TPositiveIncreasingControlInteger& TPositiveIncreasingControlInteger::operator=(const TPositiveIncreasingControlInteger& v) { if (&v != this) { - AFL_VERIFY(Value < v.Value); + AFL_VERIFY(Value < v.Value)("from", Value)("to", v.Value); Value = v.Value; } return *this; From eed1c41264cfdcba10930cc7ca79c189c400a7a1 Mon Sep 17 00:00:00 2001 From: Iurii Kravchenko Date: Wed, 2 Jul 2025 11:07:11 +0200 Subject: [PATCH 51/62] Add compaction page (#20274) --- ydb/core/tx/columnshard/columnshard_view.cpp | 39 +++++++++++++-- .../storage/optimizer/tiling/tiling.cpp | 48 ++++++++++++++++++- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_view.cpp b/ydb/core/tx/columnshard/columnshard_view.cpp index f117ec6358bc..72907ca5c5a7 100644 --- a/ydb/core/tx/columnshard/columnshard_view.cpp +++ b/ydb/core/tx/columnshard/columnshard_view.cpp @@ -17,6 +17,8 @@ class TTxMonitoring: public TTransactionBase { private: NMon::TEvRemoteHttpInfo::TPtr HttpInfoEvent; NJson::TJsonValue JsonReport = NJson::JSON_MAP; + TString RenderCompactionPage(); + TString RenderMainPage(); }; inline TString TEscapeHtml(const TString& in) { @@ -89,7 +91,7 @@ void TPrintErrorTable(TStringStream& html, THashMap> erro html << "" << errorType << " errors:
"; html << "" ""; - + for (auto [tier, queue] : errors) { while (!queue.empty()) { const auto& element = queue.front(); @@ -106,7 +108,7 @@ void TPrintErrorTable(TStringStream& html, THashMap> erro } } -void TTxMonitoring::Complete(const TActorContext& ctx) { +TString TTxMonitoring::RenderMainPage() { const auto& cgi = HttpInfoEvent->Get()->Cgi(); std::map, NJson::TJsonValue> schemaVersions; for (const auto& item : JsonReport["tables_manager"]["schema_versions"].GetArray()) { @@ -152,6 +154,8 @@ void TTxMonitoring::Complete(const TActorContext& ctx) { } } + html << "

Compaction

"; + html << "

Tiering Errors

"; auto readErrors = Self->Counters.GetEvictionCounters().TieringErrors->GetAllReadErrors(); auto writeErrors = Self->Counters.GetEvictionCounters().TieringErrors->GetAllWriteErrors(); @@ -159,7 +163,36 @@ void TTxMonitoring::Complete(const TActorContext& ctx) { TPrintErrorTable(html, readErrors, "read"); TPrintErrorTable(html, writeErrors, "write"); - ctx.Send(HttpInfoEvent->Sender, new NMon::TEvRemoteHttpInfoRes(html.Str())); + return html.Str(); +} + + +TString TTxMonitoring::RenderCompactionPage() { + TStringStream html; + const auto& cgi = HttpInfoEvent->Get()->Cgi(); + auto engine = Self->TablesManager.GetPrimaryIndexAsVerified(); + for (auto [tableId, _] : Self->TablesManager.GetTables()) { + html << "

TableId : " << tableId << "

"; + auto& compaction = engine.GetGranuleVerified(tableId).GetOptimizerPlanner(); + auto json = compaction.SerializeToJsonVisual(); + html << "
";
+        NJson::WriteJson(&html, &json, true, true, true);
+        html << "
"; + } + return html.Str(); +} + +void TTxMonitoring::Complete(const TActorContext& ctx) { + auto cgi = HttpInfoEvent->Get()->Cgi(); + auto path = HttpInfoEvent->Get()->PathInfo(); + TString htmlResult; + + if (cgi.Has("Compaction") && cgi.Get("Compaction") == "true") { + htmlResult = RenderCompactionPage(); + } else { + htmlResult = RenderMainPage(); + } + ctx.Send(HttpInfoEvent->Sender, new NMon::TEvRemoteHttpInfoRes(htmlResult)); } bool TColumnShard::OnRenderAppHtmlPage(NMon::TEvRemoteHttpInfo::TPtr ev, const TActorContext& ctx) { diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp index 6d4cc41efa1d..d37f8ffe9d48 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/tiling/tiling.cpp @@ -254,6 +254,16 @@ struct TAccumulator { Portions.erase(it); } } + + NJson::TJsonValue DoSerializeToJsonVisual() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("Portions", Portions.size()); + result.InsertValue("PortionsCompacting", Compacting.size()); + result.InsertValue("TotalBlobBytes", TotalBlobBytes); + result.InsertValue("LastCompaction", LastCompaction.ToString()); + result.InsertValue("TimeExceeded", TimeExceeded); + return result; + } }; struct TSimpleKeyCompare { @@ -437,6 +447,14 @@ struct TLevel { } return portions; } + + NJson::TJsonValue DoSerializeToJsonVisual() const { + NJson::TJsonValue result = NJson::JSON_MAP; + result.InsertValue("Portions", Portions.size()); + result.InsertValue("PortionsCompacting", Compacting.size()); + result.InsertValue("TotalBlobBytes", TotalBlobBytes); + return result; + } }; class TOptimizerPlanner : public IOptimizerPlanner, private TSettings { @@ -665,11 +683,37 @@ class TOptimizerPlanner : public IOptimizerPlanner, private TSettings { } TString DoDebugString() const override { - return "TODO: DebugString"; + return DoSerializeToJsonVisual().GetString(); } NJson::TJsonValue DoSerializeToJsonVisual() const override { - return NJson::JSON_NULL; + NJson::TJsonValue compaction_info = NJson::JSON_MAP; + compaction_info.InsertValue("1-Name", "TILING"); + + auto& settings = compaction_info.InsertValue("2-Settings", NJson::JSON_MAP); + settings.InsertValue("Factor", Factor); + settings.InsertValue("MaxLevels", MaxLevels); + settings.InsertValue("ExpectedPortionCount", ExpectedPortionCount); + settings.InsertValue("ExpectedPortionSize", ExpectedPortionSize); + settings.InsertValue("MaxAccumulateCount", MaxAccumulateCount); + settings.InsertValue("MaxAccumulatePortionSize", MaxAccumulatePortionSize); + settings.InsertValue("MaxAccumulateTime", MaxAccumulateTime.ToString()); + settings.InsertValue("MaxCompactionBytes", MaxCompactionBytes); + settings.InsertValue("FullCompactionUntilLevel", FullCompactionUntilLevel); + settings.InsertValue("FullCompactionMaxBytes", FullCompactionMaxBytes); + settings.InsertValue("CompactNextLevelEdges", CompactNextLevelEdges); + + NJson::TJsonValue& levels = compaction_info.InsertValue("3-Levels",NJson::JSON_ARRAY); + for (size_t level_i = 0; level_i < Max(Accumulator.size(), Levels.size()); ++level_i) { + NJson::TJsonValue& level = levels.AppendValue(NJson::JSON_MAP); + if (level_i < Accumulator.size()) { + level.InsertValue("Accumulator", Accumulator[level_i].DoSerializeToJsonVisual()); + } + if (level_i < Levels.size()) { + level.InsertValue("Level", Levels[level_i].DoSerializeToJsonVisual()); + } + } + return compaction_info; } private: From 6bec07157b65bb6f3338259071e3c9a4e4465eb3 Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Wed, 2 Jul 2025 18:45:38 +0200 Subject: [PATCH 52/62] cpu time has been fixed (#20495) --- .../engines/reader/common_reader/iterator/fetching.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp index 05584d69409f..ad98911f573d 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp @@ -67,6 +67,8 @@ TConclusion TFetchingScriptCursor::Execute(const std::shared_ptr resultStep = step->ExecuteInplace(source, *this); + const auto executionTime = TMonotonic::Now() - startInstant; + source->GetContext()->GetCommonContext()->GetCounters().AddExecutionDuration(executionTime); FlushDuration(TMonotonic::Now() - startInstant); if (!resultStep) { return resultStep; From e39fd0c3474a03419259463d2d9014d23f865d3f Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Mon, 7 Jul 2025 08:48:51 +0200 Subject: [PATCH 53/62] increasing wait time (#20676) --- ydb/core/tx/columnshard/engines/reader/actor/actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp index e33083072bc6..541f42287cdb 100644 --- a/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp +++ b/ydb/core/tx/columnshard/engines/reader/actor/actor.cpp @@ -76,7 +76,7 @@ void TColumnShardScan::Bootstrap(const TActorContext& ctx) { void TColumnShardScan::HandleScan(NColumnShard::TEvPrivate::TEvTaskProcessedResult::TPtr& ev) { if (ChunksLimiter.HasMore()) { - WaitTime = TInstant::Now() - StartWaitTime; + WaitTime += TInstant::Now() - StartWaitTime; } auto g = Stats->MakeGuard("task_result"); auto result = ev->Get()->ExtractResult(); From 3ce08ada56292ea8ed3f240c8791821c0b9ceffa Mon Sep 17 00:00:00 2001 From: Iurii Kravchenko Date: Mon, 7 Jul 2025 12:42:41 +0200 Subject: [PATCH 54/62] fix (#20686) --- ydb/core/tx/columnshard/engines/portions/data_accessor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp index d10a048800ac..b62e575432bf 100644 --- a/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp +++ b/ydb/core/tx/columnshard/engines/portions/data_accessor.cpp @@ -37,8 +37,8 @@ TPortionDataAccessor::TPreparedBatchData PrepareForAssembleImpl(const TPortionDa if ((it == portionData.GetRecordsVerified().end() || i < it->GetColumnId())) { if (restoreAbsent || IIndexInfo::IsSpecialColumn(i)) { columns.emplace_back(rowsCount, dataSchema.GetColumnLoaderOptional(i), resultSchema.GetColumnLoaderVerified(i)); + portionInfo.FillDefaultColumn(columns.back(), defaultSnapshot); } - portionInfo.FillDefaultColumn(columns.back(), defaultSnapshot); } if (it == portionData.GetRecordsVerified().end()) { continue; From 54a7110e463f23670ef8c5475d07d8bd435f243b Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Mon, 9 Jun 2025 12:08:11 +0300 Subject: [PATCH 55/62] Intersection tree with union-copy sets (#19427) --- .../intersection_tree/intersection_tree.h | 754 ++++++++++++ .../intersection_tree_ut.cpp | 122 ++ ydb/library/intersection_tree/ut/ya.make | 9 + ydb/library/intersection_tree/ya.make | 15 + ydb/library/union_copy_set/union_copy_set.h | 1028 +++++++++++++++++ .../union_copy_set/union_copy_set_ut.cpp | 312 +++++ ydb/library/union_copy_set/ut/ya.make | 9 + ydb/library/union_copy_set/ya.make | 15 + ydb/library/ya.make | 2 + 9 files changed, 2266 insertions(+) create mode 100644 ydb/library/intersection_tree/intersection_tree.h create mode 100644 ydb/library/intersection_tree/intersection_tree_ut.cpp create mode 100644 ydb/library/intersection_tree/ut/ya.make create mode 100644 ydb/library/intersection_tree/ya.make create mode 100644 ydb/library/union_copy_set/union_copy_set.h create mode 100644 ydb/library/union_copy_set/union_copy_set_ut.cpp create mode 100644 ydb/library/union_copy_set/ut/ya.make create mode 100644 ydb/library/union_copy_set/ya.make diff --git a/ydb/library/intersection_tree/intersection_tree.h b/ydb/library/intersection_tree/intersection_tree.h new file mode 100644 index 000000000000..e85d700f7e57 --- /dev/null +++ b/ydb/library/intersection_tree/intersection_tree.h @@ -0,0 +1,754 @@ +#pragma once +#include +#include +#include +#include + +namespace NKikimr { + + /** + * Intersection tree with unique hashable values + */ + template + class TIntersectionTree + : private TCompare + { + public: + /** + * A key that partitions a range into left and right subranges + */ + struct TPartitionKey { + TKey Key; + // When true keys equal to Key go into the left subtree + // A range like `[a, b]` translates into partition keys `{a, false}` and `{b, true}` + bool EqualGoesLeft; + + bool IsLeftInclusive() const { + return !EqualGoesLeft; + } + + bool IsRightInclusive() const { + return EqualGoesLeft; + } + + friend IOutputStream& operator<<(IOutputStream& out, const TPartitionKey& key) { + return out << "{ " << key.Key << ", " << (key.EqualGoesLeft ? "true" : "false") << " }"; + } + + friend auto operator<=>(const TPartitionKey& a, const TPartitionKey& b) + requires (std::same_as) + { + return CompareKeys(TCompare{}, a, b); + } + }; + + private: + static std::partial_ordering CompareKeys(const TCompare& cmp, const TKey& a, const TKey& b) { + return cmp(a, b); + } + + static std::partial_ordering CompareKeys(const TCompare& cmp, const TPartitionKey& a, const TPartitionKey& b) { + std::partial_ordering order = CompareKeys(cmp, a.Key, b.Key); + if (order != std::partial_ordering::equivalent) { + return order; + } + // A range like `[k, k]` partitions key space into `[-inf, k), [k, k], (k, +inf]`, where + // the left border (`{k, false}`) must be less than the right border (`{k, true}`). + // This means `{k, false} < {k, true}` or `false < true`. + if (!a.EqualGoesLeft) { + return b.EqualGoesLeft ? std::partial_ordering::less : std::partial_ordering::equivalent; + } else { + return b.EqualGoesLeft ? std::partial_ordering::equivalent : std::partial_ordering::greater; + } + } + + private: + struct TValueNode; + + using TValueMap = THashMap; + using TValueSet = TUnionCopySet; + + struct TValueNode : public TValueSet::TItem { + TValue Value; + TPartitionKey LeftKey; + TPartitionKey RightKey; + + TValueNode(const TValue& value, const TKey& leftKey, const TKey& rightKey) + : Value(value) + , LeftKey{leftKey, false} + , RightKey{rightKey, true} + {} + }; + + /** + * A node in the tree + */ + struct TNode { + TNode* Parent = nullptr; + std::unique_ptr Left; + std::unique_ptr Right; + const ui32 Prio; + ui32 UseCount = 0; + i32 LeftDelta = 0; + i32 RightDelta = 0; + i32 MaxLeftCount = 0; + i32 MaxRightCount = 0; + const TPartitionKey Key; + TValueSet LeftValues; + TValueSet RightValues; + + TNode(const TPartitionKey& key, ui32 prio) + : Prio(prio) + , Key(key) + {} + + friend IOutputStream& operator<<(IOutputStream& out, const TNode& node) { + out << "{ Key = " << node.Key << ", Prio = " << node.Prio; + out << ", LeftDelta = " << node.LeftDelta << ", MaxLeftCount = " << node.MaxLeftCount << ", Left = "; + if (node.Left) { + out << *node.Left; + } else { + out << "nullptr"; + } + out << ", RightDelta = " << node.RightDelta << ", MaxRightCount = " << node.MaxRightCount << ", Right = "; + if (node.Right) { + out << *node.Right; + } else { + out << "nullptr"; + } + return out << " }"; + } + + void SetLeft(std::unique_ptr child, i32 delta, TValueSet&& values) { + if (child) { + child->Parent = this; + } + Left = std::move(child); + LeftDelta += delta; + LeftValues += std::move(values); + RecomputeMaxLeftCount(); + } + + void SetRight(std::unique_ptr child, i32 delta, TValueSet&& values) { + if (child) { + child->Parent = this; + } + Right = std::move(child); + RightDelta += delta; + RightValues += std::move(values); + RecomputeMaxRightCount(); + } + + std::unique_ptr RemoveLeft() { + if (Left) { + Left->Parent = nullptr; + Left->LeftDelta += LeftDelta; + Left->MaxLeftCount += LeftDelta; + Left->RightDelta += LeftDelta; + Left->MaxRightCount += LeftDelta; + if (LeftValues) { + Left->LeftValues += LeftValues; + Left->RightValues += std::move(LeftValues); + } + } + LeftDelta = 0; + MaxLeftCount = 0; + LeftValues.Clear(); + return std::move(Left); + } + + std::unique_ptr RemoveRight() { + if (Right) { + Right->Parent = nullptr; + Right->LeftDelta += RightDelta; + Right->MaxLeftCount += RightDelta; + Right->RightDelta += RightDelta; + Right->MaxRightCount += RightDelta; + if (RightValues) { + Right->LeftValues += RightValues; + Right->RightValues += std::move(RightValues); + } + } + RightDelta = 0; + MaxRightCount = 0; + RightValues.Clear(); + return std::move(Right); + } + + i32 GetMaxCount() const { + return std::max(MaxLeftCount, MaxRightCount); + } + + void RecomputeMaxLeftCount() { + MaxLeftCount = LeftDelta + (Left ? Left->GetMaxCount() : i32(0)); + } + + void RecomputeMaxRightCount() { + MaxRightCount = RightDelta + (Right ? Right->GetMaxCount() : i32(0)); + } + }; + + public: + void Add(const TValue& value, const TKey& leftKey, const TKey& rightKey) { + auto res = Values.emplace( + std::piecewise_construct, + std::forward_as_tuple(value), + std::forward_as_tuple(value, leftKey, rightKey)); + Y_ABORT_UNLESS(res.second); + TValueNode* valueNode = &res.first->second; + + TNode* left = FindOrInsert(valueNode->LeftKey, RandomNumber()); + TNode* right = FindOrInsert(valueNode->RightKey, RandomNumber()); + left->UseCount++; + right->UseCount++; + AddRangeDelta(left, right, +1, valueNode); + } + + void Remove(const TValue& value) { + auto it = Values.find(value); + if (it == Values.end()) { + return; + } + TValueNode* valueNode = &it->second; + + TNode* left = Find(valueNode->LeftKey); + TNode* right = Find(valueNode->RightKey); + Y_ABORT_UNLESS(left && right); + left->UseCount--; + right->UseCount--; + + // Note: this will remove valueNode from all value sets + Values.erase(it); + + AddRangeDelta(left, right, -1); + if (left->UseCount == 0) { + DoRemove(left); + } + if (right->UseCount == 0) { + DoRemove(right); + } + } + + friend IOutputStream& operator<<(IOutputStream& out, const TIntersectionTree& tree) { + if (tree.Root) { + out << *tree.Root; + } else { + out << "{ }"; + } + return out; + } + + i32 GetMaxCount() const { + if (!Root) { + return 0; + } + + return Root->GetMaxCount(); + } + + class TFoundRange { + friend TIntersectionTree; + + public: + TFoundRange() = default; + + TFoundRange(const TNode* leftBorder, const TNode* rightBorder, i32 leftCount, i32 rightCount, bool lastLeft) + : LeftBorder(leftBorder) + , RightBorder(rightBorder) + , LeftCount(leftCount) + , RightCount(rightCount) + , LastLeft(lastLeft) + {} + + explicit operator bool() const { + return LeftBorder && RightBorder; + } + + bool HasLeftKey() const { + return LeftBorder != nullptr; + } + + const TPartitionKey& GetLeftPartitionKey() const { + Y_ABORT_UNLESS(LeftBorder); + return LeftBorder->Key; + } + + const TKey& GetLeftKey() const { + return GetLeftPartitionKey().Key; + } + + bool IsLeftInclusive() const { + return !GetLeftPartitionKey().IsLeftInclusive(); + } + + bool HasRightKey() const { + return RightBorder != nullptr; + } + + const TPartitionKey& GetRightPartitionKey() const { + Y_ABORT_UNLESS(RightBorder); + return RightBorder->Key; + } + + const TKey& GetRightKey() const { + return GetRightPartitionKey().Key; + } + + bool IsRightInclusive() const { + return GetRightPartitionKey().IsRightInclusive(); + } + + i32 GetCount() const { + return LastLeft ? (RightCount + RightBorder->LeftDelta) : (LeftCount + LeftBorder->RightDelta); + } + + template + bool ForEachValue(const TCallback& callback) { + const TNode* node = LastLeft ? RightBorder : LeftBorder; + if (!node) { + return true; + } + bool leftChild = LastLeft; + for (;;) { + const TValueSet& values = leftChild ? node->LeftValues : node->RightValues; + bool keepGoing = values.ForEachValue([&](const TValueNode* p) { + return callback(p->Value); + }); + if (!keepGoing) { + return false; + } + const TNode* parent = node->Parent; + if (!parent) { + break; + } + leftChild = parent->Left.get() == node; + Y_DEBUG_ABORT_UNLESS(leftChild || parent->Right.get() == node); + node = parent; + } + return true; + } + + TFoundRange Prev() const { + if (Y_UNLIKELY(!LeftBorder)) { + return TFoundRange(); + } + const TNode* newRight = LeftBorder; + i32 newRightCount = RightCount; + auto [newLeft, newLeftCount] = Predecessor(newRight, newRightCount); + if (newRight->Left) { + return TFoundRange(newLeft, newRight, newLeftCount, newRightCount, false); + } else { + return TFoundRange(newLeft, newRight, newLeftCount, newRightCount, true); + } + } + + TFoundRange Next() const { + if (Y_UNLIKELY(!RightBorder)) { + return TFoundRange(); + } + const TNode* newLeft = RightBorder; + i32 newLeftCount = RightCount; + auto [newRight, newRightCount] = Successor(newLeft, newLeftCount); + if (newLeft->Right) { + return TFoundRange(newLeft, newRight, newLeftCount, newRightCount, true); + } else { + return TFoundRange(newLeft, newRight, newLeftCount, newRightCount, false); + } + } + + private: + static std::pair Predecessor(const TNode* node, i32 count) { + if (node->Left) { + count += node->LeftDelta; + node = node->Left.get(); + while (node->Right) { + count += node->RightDelta; + node = node->Right.get(); + } + return {node, count}; + } else { + while (const TNode* parent = node->Parent) { + if (parent->Right.get() == node) { + count -= parent->RightDelta; + return {parent, count}; + } + Y_ABORT_UNLESS(parent->Left.get() == node); + count -= parent->RightDelta; + node = parent; + } + return {nullptr, count}; + } + } + + static std::pair Successor(const TNode* node, i32 count) { + if (node->Right) { + count += node->RightDelta; + node = node->Right.get(); + while (node->Left) { + count += node->LeftDelta; + node = node->Left.get(); + } + return {node, count}; + } else { + while (const TNode* parent = node->Parent) { + if (parent->Left.get() == node) { + count -= parent->LeftDelta; + return {parent, count}; + } + Y_ABORT_UNLESS(parent->Right.get() == node); + count -= parent->RightDelta; + node = parent; + } + return {nullptr, count}; + } + } + + private: + const TNode* LeftBorder = nullptr; + const TNode* RightBorder = nullptr; + i32 LeftCount = 0; + i32 RightCount = 0; + bool LastLeft = false; + }; + + TFoundRange FirstRange() const { + TFoundRange range; + const TNode* node = Root.get(); + i32 count = 0; + while (node) { + if (node->Left || range.LeftBorder) { + // We go left, truncating range on the right + range.RightBorder = node; + range.RightCount = count; + range.LastLeft = true; + count += node->LeftDelta; + node = node->Left.get(); + } else { + // We go right, truncating range on the left + range.LeftBorder = node; + range.LeftCount = count; + range.LastLeft = false; + count += node->RightDelta; + node = node->Right.get(); + } + } + return range; + } + + TFoundRange FindRange(const TKey& key) const { + TFoundRange range; + const TNode* node = Root.get(); + i32 count = 0; + while (node) { + auto order = CompareKeys(*this, key, node->Key.Key); + if (order < 0 || order == 0 && node->Key.EqualGoesLeft) { + // We go left, truncating range on the right + range.RightBorder = node; + range.RightCount = count; + range.LastLeft = true; + count += node->LeftDelta; + node = node->Left.get(); + } else { + // We go right, truncating range on the left + range.LeftBorder = node; + range.LeftCount = count; + range.LastLeft = false; + count += node->RightDelta; + node = node->Right.get(); + } + } + return range; + } + + TFoundRange GetMaxRange() const { + TFoundRange range; + const TNode* node = Root.get(); + i32 count = 0; + while (node) { + if (node->MaxLeftCount >= node->MaxRightCount) { + // We go left, truncating range on the right + range.RightBorder = node; + range.RightCount = count; + range.LastLeft = true; + count += node->LeftDelta; + node = node->Left.get(); + } else { + // We go right, truncating range on the left + range.LeftBorder = node; + range.LeftCount = count; + range.LastLeft = false; + count += node->RightDelta; + node = node->Right.get(); + } + } + return range; + } + + template + bool ForEachRange(const TCallback& callback) const { + auto range = FirstRange(); + while (range) { + if (!callback((const TFoundRange&)range)) { + return false; + } + range = range.Next(); + } + return true; + } + + private: + void AddRangeDelta(TNode* left, TNode* right, i32 delta, TValueNode* valueNode = nullptr) { + TNode* stop = CommonAncestor(left, right); + AddRangeDeltaLeftSpine(left, stop, delta, valueNode); + AddRangeDeltaRightSpine(right, stop, delta, valueNode); + while (TNode* parent = stop->Parent) { + if (stop == parent->Left.get()) { + parent->RecomputeMaxLeftCount(); + } else { + parent->RecomputeMaxRightCount(); + } + stop = parent; + } + } + + void AddRangeDeltaLeftSpine(TNode* node, TNode* stop, i32 delta, TValueNode* valueNode = nullptr) { + bool selected = true; + while (node != stop) { + TNode* parent = node->Parent; + if (selected) { + node->RightDelta += delta; + node->MaxRightCount += delta; + if (valueNode) { + node->RightValues.Add(valueNode); + } + } + if (node == parent->Left.get()) { + parent->RecomputeMaxLeftCount(); + selected = true; + } else { + parent->RecomputeMaxRightCount(); + selected = false; + } + node = parent; + } + } + + void AddRangeDeltaRightSpine(TNode* node, TNode* stop, i32 delta, TValueNode* valueNode = nullptr) { + bool selected = true; + while (node != stop) { + TNode* parent = node->Parent; + if (selected) { + node->LeftDelta += delta; + node->MaxLeftCount += delta; + if (valueNode) { + node->LeftValues.Add(valueNode); + } + } + if (node == parent->Right.get()) { + parent->RecomputeMaxRightCount(); + selected = true; + } else { + parent->RecomputeMaxLeftCount(); + selected = false; + } + node = parent; + } + } + + private: + TNode* CommonAncestor(TNode* a, TNode* b) { + TNode* p = a; + TNode* q = b; + // There are 2 cases: + // 1. a and b are the same depth, in which case by following parent + // pointers we will reach the common ancestor directly. + // 2. a and b are different depths, in which case either p or q + // will reach the root first, and we indirectly compute the + // difference between their depths. We loop from the shallower + // to the deeper node, and ascend it by the difference. By the + // time we switch from the deeper to the shallower node both + // p and q will be the same depth, which turns it into case 1. + // Complexity is equivalent to computing each depth first and + // then ascending to the same depth, but makes case 1 more + // efficient and doesn't need as many comparisons. + while (p != q) { + p = p->Parent; + q = q->Parent; + Y_ABORT_UNLESS(p || q); + if (!p) { + p = b; + } + if (!q) { + q = a; + } + } + return p; + } + + private: + TNode* Find(const TPartitionKey& key) { + TNode* t = Root.get(); + while (t) { + auto order = CompareKeys(*this, key, t->Key); + if (order == std::partial_ordering::equivalent) { + return t; + } + + t = order < 0 ? t->Left.get() : t->Right.get(); + } + return nullptr; + } + + TNode* FindOrInsert(const TPartitionKey& key, ui32 prio) { + std::unique_ptr l, r; + i32 lDelta = 0; + i32 rDelta = 0; + TValueSet lValues; + TValueSet rValues; + TNode* parent = nullptr; + bool leftChild = false; + std::unique_ptr* tptr = &Root; + TNode* t; + while ((t = tptr->get()) != nullptr) { + if (prio < t->Prio) { + // New node must be a parent of t, split it first + if (TNode* p = FindOrSplit(std::move(*tptr), l, lDelta, lValues, r, rDelta, rValues, key)) { + // We found an existing node for key + return p; + } + + break; + } + + auto order = CompareKeys(*this, key, t->Key); + if (order == std::partial_ordering::equivalent) { + return t; + } + + parent = t; + leftChild = order < 0; + tptr = leftChild ? &t->Left : &t->Right; + } + + // Make a new subtree root with l and r as children + // Note: parent's max count aggregate doesn't change + tptr->reset((t = new TNode(key, prio))); + t->Parent = parent; + t->SetLeft(std::move(l), lDelta, std::move(lValues)); + t->SetRight(std::move(r), rDelta, std::move(rValues)); + return t; + } + + /** + * Split subtree t into subtrees l and r, such that l < key and key < r + * + * Does nothing and returns existing node if key already exist in subtree t. + */ + TNode* FindOrSplit( + std::unique_ptr&& t, + std::unique_ptr& l, i32& lDelta, TValueSet& lValues, + std::unique_ptr& r, i32& rDelta, TValueSet& rValues, + const TPartitionKey& key) + { + if (!t) { + return nullptr; + } + + auto order = CompareKeys(*this, key, t->Key); + if (order == std::partial_ordering::equivalent) { + // Key already exists in the subtree + return t.get(); + } + + if (order < 0) { + // We must split the left subtree + lDelta += t->LeftDelta; + lValues += t->LeftValues; + if (t->Left) { + std::unique_ptr tmp; + i32 tmpDelta = 0; + TValueSet tmpValues; + if (TNode* found = FindOrSplit(std::move(t->Left), l, lDelta, lValues, tmp, tmpDelta, tmpValues, key)) { + return found; + } + t->SetLeft(std::move(tmp), tmpDelta, std::move(tmpValues)); + } + t->Parent = nullptr; + r = std::move(t); + } else { + // We must split the right subtree + rDelta += t->RightDelta; + rValues += t->RightValues; + if (t->Right) { + std::unique_ptr tmp; + i32 tmpDelta = 0; + TValueSet tmpValues; + if (TNode* found = FindOrSplit(std::move(t->Right), tmp, tmpDelta, tmpValues, r, rDelta, rValues, key)) { + return found; + } + t->SetRight(std::move(tmp), tmpDelta, std::move(tmpValues)); + } + t->Parent = nullptr; + l = std::move(t); + } + + return nullptr; + } + + private: + void DoRemove(TNode* t) { + TNode* p = t->Parent; + if (p) { + if (p->Left.get() == t) { + DoRemove(&p->Left, true); + } else { + DoRemove(&p->Right, false); + } + } else { + DoRemove(&Root, true); + } + } + + void DoRemove(std::unique_ptr* tptr, bool leftChild) { + std::unique_ptr d = std::move(*tptr); + TNode* parent = std::exchange(d->Parent, nullptr); + + // Note: parent's max count aggregate doesn't change + + if (d->Left || d->Right) { + *tptr = Merge(d->RemoveLeft(), d->RemoveRight()); + tptr->get()->Parent = parent; + } else if (parent) { + // Y_ABORT_UNLESS(d->LeftDelta == d->RightDelta); + if (leftChild) { + parent->LeftDelta += d->LeftDelta; + parent->LeftValues += std::move(d->LeftValues); + } else { + parent->RightDelta += d->RightDelta; + parent->RightValues += std::move(d->RightValues); + } + } + } + + std::unique_ptr Merge(std::unique_ptr l, std::unique_ptr r) { + if (!l || !r) { + if (l) { + return l; + } else { + return r; + } + } else if (l->Prio <= r->Prio) { + l->SetRight(Merge(l->RemoveRight(), std::move(r)), 0, {}); + return l; + } else { + r->SetLeft(Merge(std::move(l), r->RemoveLeft()), 0, {}); + return r; + } + } + + private: + std::unique_ptr Root; + TValueMap Values; + }; + +} // namespace NKikimr diff --git a/ydb/library/intersection_tree/intersection_tree_ut.cpp b/ydb/library/intersection_tree/intersection_tree_ut.cpp new file mode 100644 index 000000000000..03e7d1ea6e56 --- /dev/null +++ b/ydb/library/intersection_tree/intersection_tree_ut.cpp @@ -0,0 +1,122 @@ +#include "intersection_tree.h" + +#include +#include +#include + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(IntersectionTree) { + + struct TExampleRange { + ui64 MinKey; + ui64 MaxKey; + }; + + using TExampleTree = TIntersectionTree; + + std::vector GenerateRanges(size_t count, ui64 maxValue) { + std::vector ranges; + ranges.reserve(count); + for (size_t i = 0; i < count; ++i) { + ui64 minKey = RandomNumber() % maxValue; + ui64 maxKey = minKey + RandomNumber() % (maxValue - minKey); + ranges.push_back({ minKey, maxKey }); + } + return ranges; + } + + struct TDebugRange { + TExampleTree::TPartitionKey Left; + TExampleTree::TPartitionKey Right; + i32 Count; + }; + + TString ToString(const std::vector& ranges) { + TStringBuilder b; + for (const auto& range : ranges) { + b << (range.Left.EqualGoesLeft ? '(' : '[') << range.Left.Key + << ", " << range.Right.Key << (range.Right.EqualGoesLeft ? ']' : ')') + << " = " << range.Count << Endl; + } + return b; + } + + std::vector TreeToRanges(const TExampleTree& tree) { + std::vector ranges; + tree.ForEachRange([&](const auto& range) { + ranges.push_back({ range.GetLeftPartitionKey(), range.GetRightPartitionKey(), range.GetCount() }); + return true; + }); + return ranges; + } + + std::vector SpanToRanges(const std::span& srcRanges) { + std::map deltas; + for (const auto& range : srcRanges) { + deltas[{ range.MinKey, false}] += 1; + deltas[{ range.MaxKey, true}] -= 1; + } + std::vector ranges; + std::optional lastKey; + i32 count = 0; + for (const auto& [key, delta] : deltas) { + if (lastKey) { + ranges.push_back({ *lastKey, key, count }); + } + lastKey = key; + count += delta; + } + return ranges; + } + + void CheckMaxRange(const TExampleTree& tree) { + auto maxCount = tree.GetMaxCount(); + auto maxRange = tree.GetMaxRange(); + if (maxCount > 0) { + UNIT_ASSERT(maxRange.HasLeftKey() && maxRange.HasRightKey()); + UNIT_ASSERT_VALUES_EQUAL(maxRange.GetCount(), maxCount); + i32 count = 0; + bool unique = true; + THashSet visited; + maxRange.ForEachValue([&](size_t value) { + ++count; + if (!visited.insert(value).second) { + unique = false; + } + return true; + }); + UNIT_ASSERT_VALUES_EQUAL(count, maxRange.GetCount()); + UNIT_ASSERT(unique); + } + } + + Y_UNIT_TEST(Randomized) { + size_t removeCount = 99; + auto rangesVec = GenerateRanges(100, 1000000); + std::span ranges(rangesVec.data(), rangesVec.size()); + TExampleTree tree; + size_t nextPosition = 0; + for (const auto& range : ranges) { + tree.Add(nextPosition++, range.MinKey, range.MaxKey); + } + UNIT_ASSERT_VALUES_EQUAL( + ToString(TreeToRanges(tree)), + ToString(SpanToRanges(ranges)) + ); + CheckMaxRange(tree); + nextPosition = 0; + for (size_t i = 0; i < removeCount && !ranges.empty(); ++i) { + tree.Remove(nextPosition++); + ranges = ranges.subspan(1); + UNIT_ASSERT_VALUES_EQUAL( + ToString(TreeToRanges(tree)), + ToString(SpanToRanges(ranges)) + ); + CheckMaxRange(tree); + } + } + +} // Y_UNIT_TEST_SUITE(IntersectionTree) + +} // namespace NKikimr diff --git a/ydb/library/intersection_tree/ut/ya.make b/ydb/library/intersection_tree/ut/ya.make new file mode 100644 index 000000000000..5f6feec6cd43 --- /dev/null +++ b/ydb/library/intersection_tree/ut/ya.make @@ -0,0 +1,9 @@ +UNITTEST_FOR(ydb/library/intersection_tree) + +SIZE(MEDIUM) + +SRCS( + intersection_tree_ut.cpp +) + +END() diff --git a/ydb/library/intersection_tree/ya.make b/ydb/library/intersection_tree/ya.make new file mode 100644 index 000000000000..63995dae61a3 --- /dev/null +++ b/ydb/library/intersection_tree/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +PEERDIR( + ydb/library/union_copy_set +) + +SRCS( + intersection_tree.h +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/union_copy_set/union_copy_set.h b/ydb/library/union_copy_set/union_copy_set.h new file mode 100644 index 000000000000..0a0863975d4d --- /dev/null +++ b/ydb/library/union_copy_set/union_copy_set.h @@ -0,0 +1,1028 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace NKikimr { + + /** + * Union-Copy Disjoint Set data structure + * + * Has O(1) union and O(1) copy operations for sets + */ + template + class TUnionCopySet { + public: + /** + * Base class for TValue + */ + class TItem; + + private: + /** + * Pointer tag that specifies the target type + */ + enum class ETag : uint8_t { + // Pointer is to either set or item + Item = 0, + // Pointer is to the copy node/item + Copy = 1 << 0, + // Pointer is to the union node/item + Union = 1 << 1, + }; + + static constexpr uintptr_t TagMask = uintptr_t(3); + static constexpr uintptr_t PointerMask = ~TagMask; + + struct TCopyNodeItem; + struct TCopyNode; + struct TUnionNodeItem; + struct TUnionNode; + + /** + * A tagged pointer that is pointing up (towards sets) + */ + class TUpPtr { + public: + explicit operator bool() const { + return Value != 0; + } + + ETag GetTag() const { + return ETag(Value & TagMask); + } + + bool IsItemPtr() const { + return GetTag() == ETag::Item; + } + + TUnionCopySet* GetItemPtr() const { + Y_ABORT_UNLESS(IsItemPtr()); + return reinterpret_cast(Value); + } + + bool IsCopyNodePtr() const { + return GetTag() == ETag::Copy; + } + + TCopyNode* GetCopyNodePtr() const { + Y_ABORT_UNLESS(IsCopyNodePtr()); + return reinterpret_cast(Value & PointerMask); + } + + bool IsUnionNodeItemPtr() const { + return GetTag() == ETag::Union; + } + + TUnionNodeItem* GetUnionNodeItemPtr() const { + Y_ABORT_UNLESS(IsUnionNodeItemPtr()); + return reinterpret_cast(Value & PointerMask); + } + + void SetNull() { + Value = 0; + } + + void Set(TUnionCopySet* p) { + Value = reinterpret_cast(p); + } + + void Set(TCopyNode* p) { + Value = reinterpret_cast(p) | uintptr_t(ETag::Copy); + } + + void Set(TUnionNodeItem* p) { + Value = reinterpret_cast(p) | uintptr_t(ETag::Union); + } + + friend bool operator==(const TUpPtr& a, TUpPtr& b) noexcept { + return a.Value == b.Value; + } + + friend bool operator==(const TUpPtr& a, TUnionCopySet* b) noexcept { + return a.IsItemPtr() && a.GetItemPtr() == b; + } + + friend bool operator==(const TUpPtr& a, TCopyNode* b) noexcept { + return a.IsCopyNodePtr() && a.GetCopyNodePtr() == b; + } + + friend bool operator==(const TUpPtr& a, TUnionNodeItem* b) noexcept { + return a.IsUnionNodeItemPtr() && a.GetUnionNodeItemPtr() == b; + } + + private: + uintptr_t Value = 0; + }; + + /** + * A tagged pointer that is pointing down (towards items) + */ + class TDownPtr { + public: + explicit operator bool() const { + return Value != 0; + } + + ETag GetTag() const { + return ETag(Value & TagMask); + } + + bool IsItemPtr() const { + return GetTag() == ETag::Item; + } + + TItem* GetItemPtr() const { + Y_ABORT_UNLESS(IsItemPtr()); + return reinterpret_cast(Value); + } + + bool IsCopyNodeItemPtr() const { + return GetTag() == ETag::Copy; + } + + TCopyNodeItem* GetCopyNodeItemPtr() const { + Y_ABORT_UNLESS(IsCopyNodeItemPtr()); + return reinterpret_cast(Value & PointerMask); + } + + bool IsUnionNodePtr() const { + return GetTag() == ETag::Union; + } + + TUnionNode* GetUnionNodePtr() const { + Y_ABORT_UNLESS(IsUnionNodePtr()); + return reinterpret_cast(Value & PointerMask); + } + + void SetNull() { + Value = 0; + } + + void Set(TItem* p) { + Value = reinterpret_cast(p); + } + + void Set(TCopyNodeItem* p) { + Value = reinterpret_cast(p) | uintptr_t(ETag::Copy); + } + + void Set(TUnionNode* p) { + Value = reinterpret_cast(p) | uintptr_t(ETag::Union); + } + + friend bool operator==(const TDownPtr& a, const TDownPtr& b) noexcept { + return a.Value == b.Value; + } + + friend bool operator==(const TDownPtr& a, TItem* p) noexcept { + return a.IsItemPtr() && a.GetItemPtr() == p; + } + + friend bool operator==(const TDownPtr& a, TCopyNodeItem* b) noexcept { + return a.IsCopyNodeItemPtr() && a.GetCopyNodeItemPtr() == b; + } + + friend bool operator==(const TDownPtr& a, TUnionNode* b) noexcept { + return a.IsUnionNodePtr() && a.GetUnionNodePtr() == b; + } + + private: + uintptr_t Value = 0; + }; + + struct TCopyNodeItem : public TIntrusiveListItem { + TUpPtr Up; + TCopyNode* Down = nullptr; + }; + + struct TCopyNode { + TIntrusiveList Items; + size_t Count = 0; + TDownPtr Down; + + void Add(TCopyNodeItem* p) { + Items.PushBack(p); + p->Down = this; + ++Count; + } + + void Remove(TCopyNodeItem* p) { + p->Down = nullptr; + Items.Remove(p); + --Count; + } + }; + + struct TUnionNodeItem : public TIntrusiveListItem { + TUnionNode* Up = nullptr; + TDownPtr Down; + }; + + struct TUnionNode { + TIntrusiveList Items; + size_t Count = 0; + TUpPtr Up; + + void Add(TUnionNodeItem* p) { + Items.PushBack(p); + p->Up = this; + ++Count; + } + + void Remove(TUnionNodeItem* p) { + p->Up = nullptr; + Items.Remove(p); + --Count; + } + }; + + public: + class TItem { + friend TUnionCopySet; + + public: + TItem() noexcept = default; + + TItem(TItem&& rhs) noexcept + : Up(rhs.Up) + { + if (Up) { + rhs.Up.SetNull(); + RelinkUpToSelf(); + } + } + + ~TItem() { + if (Up) { + DestroyUp(Up); + } + } + + TItem(const TItem&) = delete; + TItem& operator=(const TItem&) = delete; + + private: + void RelinkUpToSelf() noexcept { + switch (Up.GetTag()) { + case ETag::Item: { + Up.GetItemPtr()->Down.Set(this); + break; + } + case ETag::Copy: { + Up.GetCopyNodePtr()->Down.Set(this); + break; + } + case ETag::Union: { + Up.GetUnionNodeItemPtr()->Down.Set(this); + break; + } + } + } + + private: + TUpPtr Up; + }; + + private: + static void DestroyUp(TUpPtr up) { + // Note: we cannot actually have a deep recursion in an optimized tree, + // because copy and union nodes alternate, and when we encounter a + // union node it has at least 2 items. When a single item is left + // it is optimized away without recursion. However, we might want + // to skip expensive optimization passes in the future, and might + // have a potentially deep chain of copy nodes, so avoid recursion. + TStackVec queue; + for (;;) { + switch (up.GetTag()) { + case ETag::Item: { + TUnionCopySet* set = up.GetItemPtr(); + set->Down.SetNull(); + break; + } + case ETag::Copy: { + TCopyNode* node = up.GetCopyNodePtr(); + if (node->Items.Empty()) { + // This node is empty and can finally be deleted + std::unique_ptr d(node); + break; + } + // Mark this node as "delete in progress" so when a set + // is not disjoint this node is not optimized away and + // deleted unexpectedly (since we still hold a pointer + // to it and need it to remain valid) + node->Count = 0; + node->Down.SetNull(); + // We want to visit this node again until it's empty + queue.push_back(up); + // Remove one item at a time + std::unique_ptr item(node->Items.PopFront()); + Y_DEBUG_ABORT_UNLESS(item->Up, "Unexpected null link up during destruction"); + if (item->Up) [[likely]] { + up = item->Up; + continue; + } + break; + } + case ETag::Union: { + // We remove current item from the union node + std::unique_ptr item(up.GetUnionNodeItemPtr()); + TUnionNode* node = item->Up; + Y_DEBUG_ABORT_UNLESS(node->Count > 0); + node->Remove(item.get()); + item.reset(); + switch (node->Count) { + case 0: [[unlikely]] { + // Note: this cannot normally happen, since nodes + // are removed from the graph when their count + // reaches 1. However allocation errors during + // insertions could leave count 1 in place. + std::unique_ptr d(node); + Y_DEBUG_ABORT_UNLESS(node->Up, "Unexpected null link up during destruction"); + if (node->Up) [[likely]] { + up = node->Up; + continue; + } + break; + } + case 1: { + OptimizeAway(node); + break; + } + } + break; + } + } + if (queue.empty()) { + break; + } + up = queue.back(); + queue.pop_back(); + } + } + + static void DestroyDown(TDownPtr down) { + // Note: we cannot actually have a deep recursion in an optimized tree, + // because copy and union nodes alternate, and when we encounter a + // union node it has at least 2 items. When a single item is left + // it is optimized away without recursion. However, we might want + // to skip expensive optimization passes in the future, and might + // have a potentially deep chain of copy nodes, so avoid recursion. + TStackVec queue; + for (;;) { + switch (down.GetTag()) { + case ETag::Item: { + TItem* item = down.GetItemPtr(); + item->Up.SetNull(); + break; + } + case ETag::Copy: { + // We remove current item from the copy node + std::unique_ptr item(down.GetCopyNodeItemPtr()); + TCopyNode* node = item->Down; + Y_DEBUG_ABORT_UNLESS(node->Count > 0); + node->Remove(item.get()); + item.reset(); + switch (node->Count) { + case 0: [[unlikely]] { + // Note: this cannot normally happen, since nodes + // are removed from the graph when their count + // reaches 1. However allocation errors during + // insertions could leave count 1 in place. + std::unique_ptr d(node); + Y_DEBUG_ABORT_UNLESS(node->Down, "Unexpected null link down during destruction"); + if (node->Down) [[likely]] { + down = node->Down; + continue; + } + break; + } + case 1: { + OptimizeAway(node); + break; + } + } + break; + } + case ETag::Union: { + TUnionNode* node = down.GetUnionNodePtr(); + if (node->Items.Empty()) { + // This node is empty and can finally be deleted + std::unique_ptr d(node); + break; + } + // Mark this node as "delete in progress" so when a set + // is not disjoint this node is not optimized away and + // deleted unexpectedly (since we still hold a pointer + // to it and need it to remain valid) + node->Count = 0; + node->Up.SetNull(); + // We want to visit this node again until it's empty + queue.push_back(down); + // Remove one item at a time + std::unique_ptr item(node->Items.PopFront()); + Y_DEBUG_ABORT_UNLESS(item->Down, "Unexpected null link down during destruction"); + if (item->Down) [[likely]] { + down = item->Down; + continue; + } + break; + } + } + if (queue.empty()) { + break; + } + down = queue.back(); + queue.pop_back(); + } + } + + template + static void RelinkPair(TParent* parent, TDownPtr child) { + parent->Down = child; + switch (child.GetTag()) { + case ETag::Item: { + TItem* item = child.GetItemPtr(); + item->Up.Set(parent); + break; + } + case ETag::Copy: { + TCopyNodeItem* item = child.GetCopyNodeItemPtr(); + item->Up.Set(parent); + break; + } + case ETag::Union: { + TUnionNode* node = child.GetUnionNodePtr(); + node->Up.Set(parent); + break; + } + } + } + + template + static void RelinkPair(TUpPtr parent, TChild* child) { + child->Up = parent; + switch (parent.GetTag()) { + case ETag::Item: { + TUnionCopySet* set = parent.GetItemPtr(); + set->Down.Set(child); + break; + } + case ETag::Copy: { + TCopyNode* node = parent.GetCopyNodePtr(); + node->Down.Set(child); + break; + } + case ETag::Union: { + TUnionNodeItem* item = parent.GetUnionNodeItemPtr(); + item->Down.Set(child); + break; + } + } + } + + template + static void OptimizePair(TParent* parent, TChild* child) { + parent->Down.Set(child); + child->Up.Set(parent); + } + + static void OptimizePair(TCopyNode* parent, TCopyNodeItem* child) { + parent->Down.Set(child); + child->Up.Set(parent); + TCopyNode* bottom = child->Down; + if (parent->Count == 0 || bottom->Count == 0) { + // Either parent or child are going to be deleted soon + return; + } + // We currently have: parent items <-> parent <-> child <-> bottom + // And we need to remove the redundant `child` item + bottom->Remove(child); + std::unique_ptr dChild(child); + // The smaller number of items are folded into the larger node + if (parent->Count <= bottom->Count) { + // Move all parent items to bottom + parent->Count = 0; + std::unique_ptr dParent(parent); + while (!parent->Items.Empty()) { + TCopyNodeItem* item = parent->Items.PopFront(); + bottom->Add(item); + } + } else { + // Move all bottom items to parent + bottom->Count = 0; + RelinkPair(parent, bottom->Down); + std::unique_ptr dBottom(bottom); + while (!bottom->Items.Empty()) { + TCopyNodeItem* item = bottom->Items.PopFront(); + parent->Add(item); + } + } + } + + static void OptimizePair(TUnionNodeItem* parent, TUnionNode* child) { + parent->Down.Set(child); + child->Up.Set(parent); + TUnionNode* top = parent->Up; + if (top->Count == 0 || child->Count == 0) { + // Either parent or child are going to be deleted soon + return; + } + // We currently have top <-> parent <-> child -> child items + // And we need to remove the redundant `parent` item + top->Remove(parent); + std::unique_ptr dParent(parent); + // The smaller number of items are folded into the larger node + if (child->Count <= top->Count) { + // Move all child items to top + child->Count = 0; + std::unique_ptr dChild(child); + while (!child->Items.Empty()) { + TUnionNodeItem* item = child->Items.PopFront(); + top->Add(item); + } + } else { + // Move all top items to child + top->Count = 0; + RelinkPair(top->Up, child); + std::unique_ptr dTop(top); + while (!top->Items.Empty()) { + TUnionNodeItem* item = top->Items.PopFront(); + child->Add(item); + } + } + } + + template + static void OptimizePair(TParent* parent, TDownPtr down) { + Y_ABORT_UNLESS(down); + switch (down.GetTag()) { + case ETag::Item: { + TItem* child = down.GetItemPtr(); + OptimizePair(parent, child); + break; + } + case ETag::Copy: { + TCopyNodeItem* child = down.GetCopyNodeItemPtr(); + OptimizePair(parent, child); + break; + } + case ETag::Union: { + TUnionNode* child = down.GetUnionNodePtr(); + OptimizePair(parent, child); + break; + } + } + } + + static void OptimizePair(TUpPtr up, TDownPtr down) { + Y_ABORT_UNLESS(up); + switch (up.GetTag()) { + case ETag::Item: { + TUnionCopySet* parent = up.GetItemPtr(); + OptimizePair(parent, down); + break; + } + case ETag::Copy: { + TCopyNode* parent = up.GetCopyNodePtr(); + OptimizePair(parent, down); + break; + } + case ETag::Union: { + TUnionNodeItem* parent = up.GetUnionNodeItemPtr(); + OptimizePair(parent, down); + break; + } + } + } + + static void OptimizeAway(TCopyNode* node) { + Y_ABORT_UNLESS(node->Count == 1); + Y_ABORT_UNLESS(!node->Items.Empty()); + std::unique_ptr d(node); + std::unique_ptr nodeItem(node->Items.PopFront()); + Y_ABORT_UNLESS(node->Items.Empty()); + node->Count = 0; + OptimizePair(nodeItem->Up, node->Down); + } + + static void OptimizeAway(TUnionNode* node) { + Y_ABORT_UNLESS(node->Count == 1); + Y_ABORT_UNLESS(!node->Items.Empty()); + std::unique_ptr d(node); + std::unique_ptr nodeItem(node->Items.PopFront()); + Y_ABORT_UNLESS(node->Items.Empty()); + node->Count = 0; + OptimizePair(node->Up, nodeItem->Down); + } + + private: + template + static TCopyNode* InsertCopyNode(TParent* parent, TChild* child) { + std::unique_ptr nodeItem(new TCopyNodeItem); + std::unique_ptr node(new TCopyNode); + // parent <-> nodeItem <-> node <-> child + node->Add(nodeItem.get()); + nodeItem->Up.Set(parent); + node->Down.Set(child); + parent->Down.Set(nodeItem.release()); + child->Up.Set(node.get()); + return node.release(); + } + + template + static TUnionNode* InsertUnionNode(TParent* parent, TChild* child) { + std::unique_ptr nodeItem(new TUnionNodeItem); + std::unique_ptr node(new TUnionNode); + // parent <-> node <-> nodeItem <-> child + node->Add(nodeItem.get()); + nodeItem->Down.Set(child); + node->Up.Set(parent); + child->Up.Set(nodeItem.release()); + parent->Down.Set(node.get()); + return node.release(); + } + + static TCopyNode* EnsureCopyNode(TUnionCopySet* set) { + switch (set->Down.GetTag()) { + case ETag::Item: { + return InsertCopyNode(set, set->Down.GetItemPtr()); + } + case ETag::Copy: { + return set->Down.GetCopyNodeItemPtr()->Down; + } + case ETag::Union: { + return InsertCopyNode(set, set->Down.GetUnionNodePtr()); + } + } + } + + static TUnionNode* EnsureUnionNode(TUnionCopySet* set) { + switch (set->Down.GetTag()) { + case ETag::Item: { + return InsertUnionNode(set, set->Down.GetItemPtr()); + } + case ETag::Copy: { + return InsertUnionNode(set, set->Down.GetCopyNodeItemPtr()); + } + case ETag::Union: { + return set->Down.GetUnionNodePtr(); + } + } + } + + static TCopyNode* EnsureCopyNode(TItem* item) { + switch (item->Up.GetTag()) { + case ETag::Item: { + return InsertCopyNode(item->Up.GetItemPtr(), item); + } + case ETag::Copy: { + return item->Up.GetCopyNodePtr(); + } + case ETag::Union: { + return InsertCopyNode(item->Up.GetUnionNodeItemPtr(), item); + } + } + } + + private: + void RelinkDownToSelf() { + switch (Down.GetTag()) { + case ETag::Item: { + Down.GetItemPtr()->Up.Set(this); + break; + } + case ETag::Copy: { + Down.GetCopyNodeItemPtr()->Up.Set(this); + break; + } + case ETag::Union: { + Down.GetUnionNodePtr()->Up.Set(this); + break; + } + } + } + + public: + TUnionCopySet() noexcept = default; + + ~TUnionCopySet() { + Clear(); + } + + void Clear() noexcept { + if (Down) { + DestroyDown(Down); + Down.SetNull(); + } + } + + TUnionCopySet(TUnionCopySet&& rhs) noexcept { + *this = std::move(rhs); + } + + TUnionCopySet& operator=(TUnionCopySet&& rhs) noexcept { + if (this != &rhs) { + Clear(); + Down = rhs.Down; + if (Down) { + rhs.Down.SetNull(); + RelinkDownToSelf(); + } + } + return *this; + } + + TUnionCopySet(const TUnionCopySet& rhs) { + *this = rhs; + } + + TUnionCopySet& operator=(const TUnionCopySet& rhs) { + if (this != &rhs) { + Clear(); + if (rhs.Down) { + std::unique_ptr nodeItem(new TCopyNodeItem); + TCopyNode* node = EnsureCopyNode(const_cast(&rhs)); + node->Add(nodeItem.get()); + nodeItem->Up.Set(this); + Down.Set(nodeItem.release()); + } + } + return *this; + } + + bool Empty() const { + return !Down; + } + + explicit operator bool() const { + return !Empty(); + } + + TUnionCopySet& Add(TValue* value) { + // Must be implicitly castable + TItem* item = value; + if (!Down && !item->Up) { + // First time set/item connection + item->Up.Set(this); + Down.Set(item); + } else if (!item->Up) { + // First time item is connected anywhere + std::unique_ptr nodeItem(new TUnionNodeItem); + TUnionNode* node = EnsureUnionNode(this); + // node <-> nodeItem <-> item + node->Add(nodeItem.get()); + nodeItem->Down.Set(item); + item->Up.Set(nodeItem.release()); + } else if (!Down) { + // First time set is connected anywhere + std::unique_ptr nodeItem(new TCopyNodeItem); + TCopyNode* node = EnsureCopyNode(item); + // set <-> nodeItem <-> node + node->Add(nodeItem.get()); + nodeItem->Up.Set(this); + Down.Set(nodeItem.release()); + } else { + // Complex case where union to copy connection is needed + std::unique_ptr unionItem(new TUnionNodeItem); + std::unique_ptr copyItem(new TCopyNodeItem); + TUnionNode* unionNode = EnsureUnionNode(this); + TCopyNode* copyNode = EnsureCopyNode(value); + // unionNode <-> unionItem <-> copyItem <-> copyNode + unionItem->Down.Set(copyItem.get()); + copyItem->Up.Set(unionItem.get()); + unionNode->Add(unionItem.release()); + copyNode->Add(copyItem.release()); + } + return *this; + } + + TUnionCopySet& Add(const TUnionCopySet& rhs) { + if (this != &rhs) { + if (!rhs.Down) { + // Nothing to add + } else if (!Down) { + // Adding to an empty set, make a copy + std::unique_ptr nodeItem(new TCopyNodeItem); + TCopyNode* node = EnsureCopyNode(const_cast(&rhs)); + // this <-> nodeItem <-> node + node->Add(nodeItem.get()); + nodeItem->Up.Set(this); + Down.Set(nodeItem.release()); + } else { + // Make a copy and add it to a union + std::unique_ptr unionItem(new TUnionNodeItem); + std::unique_ptr copyItem(new TCopyNodeItem); + TUnionNode* unionNode = EnsureUnionNode(this); + TCopyNode* copyNode = EnsureCopyNode(const_cast(&rhs)); + // unionNode <-> unionItem <-> copyItem <-> copyNode + unionItem->Down.Set(copyItem.get()); + copyItem->Up.Set(unionItem.get()); + unionNode->Add(unionItem.release()); + copyNode->Add(copyItem.release()); + } + } + return *this; + } + + TUnionCopySet& operator+=(const TUnionCopySet& rhs) { + return Add(rhs); + } + + TUnionCopySet& Add(TUnionCopySet&& rhs) { + if (this != &rhs) { + if (!rhs.Down) { + // Nothing to add/move + } else if (!Down) { + // Add/moving to an empty set, this is a move + *this = std::move(rhs); + } else { + // Move rhs's target to a union, unless it's already a union + std::unique_ptr unionItem(new TUnionNodeItem); + // Special case add/moving for unions (avoid generating extra unions) + if (rhs.Down.IsUnionNodePtr()) { + TUnionNode* unionNode = rhs.Down.GetUnionNodePtr(); + if (Down.IsUnionNodePtr()) { + // Add rhs union to our union, but then optimize them + TUnionNode* top = Down.GetUnionNodePtr(); + top->Add(unionItem.get()); + rhs.Down.SetNull(); + // This will link and optimize nodes + OptimizePair(unionItem.release(), unionNode); + } else { + // Move current item to rhs union and then move that union here + RelinkPair(unionItem.get(), Down); + Down.SetNull(); + unionNode->Add(unionItem.release()); + *this = std::move(rhs); + } + } else { + // Ensure this is a union and add rhs's item there + TUnionNode* unionNode = EnsureUnionNode(this); + RelinkPair(unionItem.get(), rhs.Down); + rhs.Down.SetNull(); + unionNode->Add(unionItem.release()); + } + } + } + return *this; + } + + TUnionCopySet& operator+=(TUnionCopySet&& rhs) { + return Add(std::move(rhs)); + } + + public: + template + bool ForEachValue(const TCallback& callback) const { + return ForEachValueImpl(Down, callback); + } + + private: + struct TForEachValueState { + TDownPtr Ptr; + const TUnionNodeItem* Item = nullptr; + }; + + template + static bool ForEachValueImpl(TDownPtr down, const TCallback& callback) { + if (!down) { + return true; + } + TForEachValueState state = { down }; + TStackVec queue; + for (;;) { + switch (state.Ptr.GetTag()) { + case ETag::Item: { + const TItem* item = state.Ptr.GetItemPtr(); + if (!callback(static_cast(item))) { + return false; + } + break; + } + case ETag::Copy: { + const TCopyNodeItem* item = state.Ptr.GetCopyNodeItemPtr(); + const TCopyNode* node = item->Down; + state = { node->Down }; + continue; + } + case ETag::Union: { + const TUnionNode* node = state.Ptr.GetUnionNodePtr(); + const TUnionNodeItem* item = state.Item; // next item + if (!item) { + if (node->Items.Empty()) { + // Shouldn't happen + break; + } + item = node->Items.Front(); + } + typename TIntrusiveList::const_iterator it(item); + if (++it != node->Items.End()) { + queue.push_back({ state.Ptr, &*it }); + } + state = { item->Down }; + continue; + } + } + if (queue.empty()) { + break; + } + state = queue.back(); + queue.pop_back(); + } + return true; + } + + public: + TString DebugString(bool withAddresses = false) const { + TStringBuilder sb; + DebugPrint(sb.Out, withAddresses); + return sb; + } + + void DebugPrint(IOutputStream& out, bool withAddresses = false) const { + if (withAddresses) { + out << "Set(" << (const void*)this << ") -> "; + } + DebugPrintImpl(Down, out, withAddresses); + } + + private: + static void DebugPrintImpl(TDownPtr down, IOutputStream& out, bool withAddresses) { + if (!down) { + out << "empty"; + return; + } + + TForEachValueState state = { down }; + TStackVec queue; + for (;;) { + switch (state.Ptr.GetTag()) { + case ETag::Item: { + const TItem* item = state.Ptr.GetItemPtr(); + const TValue* value = static_cast(item); + if (withAddresses) { + out << "Value(" << (const void*)value << ") = "; + } + out << *value; + break; + } + case ETag::Copy: { + const TCopyNodeItem* item = state.Ptr.GetCopyNodeItemPtr(); + const TCopyNode* node = item->Down; + if (withAddresses) { + out << "CopyNodeItem(" << (const void*)item << ") -> "; + out << "CopyNode(" << (const void*)node << ") -> "; + } else { + out << "Copy -> "; + } + state = { node->Down }; + continue; + } + case ETag::Union: { + const TUnionNode* node = state.Ptr.GetUnionNodePtr(); + const TUnionNodeItem* item = state.Item; // last item + if (!item) { + if (withAddresses) { + out << "UnionNode(" << (const void*)node << "){ "; + } else { + out << "Union{ "; + } + if (node->Items.Empty()) { + // Shouldn't happen + out << "}"; + break; + } + item = node->Items.Front(); + } else { + typename TIntrusiveList::const_iterator it(item); + if (++it == node->Items.End()) { + out << " }"; + break; + } + out << ", "; + item = &*it; + } + if (withAddresses) { + out << "UnionNodeItem(" << (const void*)&item << ") -> "; + } + queue.push_back({ state.Ptr, item }); + state = { item->Down }; + continue; + } + } + if (queue.empty()) { + break; + } + state = queue.back(); + queue.pop_back(); + } + } + + private: + TDownPtr Down; + }; + +} // namespace NKikimr diff --git a/ydb/library/union_copy_set/union_copy_set_ut.cpp b/ydb/library/union_copy_set/union_copy_set_ut.cpp new file mode 100644 index 000000000000..25ecbc6c7edb --- /dev/null +++ b/ydb/library/union_copy_set/union_copy_set_ut.cpp @@ -0,0 +1,312 @@ +#include "union_copy_set.h" + +#include +#include + +#include + +namespace NKikimr { + +Y_UNIT_TEST_SUITE(UnionCopySet) { + + struct TMyValue : public TUnionCopySet::TItem { + size_t Tag; + + explicit TMyValue(size_t tag = 0) + : Tag(tag) + {} + + friend IOutputStream& operator<<(IOutputStream& out, const TMyValue& value) { + out << value.Tag; + return out; + } + }; + + std::unordered_set Enumerate(const TUnionCopySet& s) { + std::unordered_set values; + s.ForEachValue([&](const TMyValue* value) { + values.insert(value); + return true; + }); + return values; + } + + size_t Count(const TUnionCopySet& s) { + size_t count = 0; + s.ForEachValue([&](const TMyValue*) { + ++count; + return true; + }); + return count; + } + + std::unordered_set MakeSet(std::initializer_list l) { + return std::unordered_set(l); + } + + Y_UNIT_TEST(Simple) { + TMyValue a, b, c; + TUnionCopySet set1, set2, set3; + set1.Add(&a); + set2.Add(&b); + set3.Add(set2); + set3.Add(&c); + UNIT_ASSERT(Enumerate(set1) == MakeSet({&a})); + UNIT_ASSERT(Enumerate(set2) == MakeSet({&b})); + UNIT_ASSERT(Enumerate(set3) == MakeSet({&b, &c})); + set2.Clear(); + UNIT_ASSERT(Enumerate(set2) == MakeSet({})); + UNIT_ASSERT(Enumerate(set3) == MakeSet({&b, &c})); + auto set4 = set3; + UNIT_ASSERT(Enumerate(set4) == MakeSet({&b, &c})); + set4.Add(&a); + UNIT_ASSERT(Enumerate(set4) == MakeSet({&a, &b, &c})); + auto set5 = std::move(set4); + UNIT_ASSERT(Enumerate(set4) == MakeSet({})); + UNIT_ASSERT(Enumerate(set5) == MakeSet({&a, &b, &c})); + set4.Add(&a); + set4 = set5; + UNIT_ASSERT(Enumerate(set4) == MakeSet({&a, &b, &c})); + set4.Clear(); + set4.Add(&a); + set4 = std::move(set5); + UNIT_ASSERT(Enumerate(set4) == MakeSet({&a, &b, &c})); + UNIT_ASSERT(Enumerate(set5) == MakeSet({})); + } + + Y_UNIT_TEST(ItemAddedToManySets) { + TMyValue value; + TUnionCopySet set1, set2, set3; + set1.Add(&value); + set2.Add(&value); + set3.Add(&value); + UNIT_ASSERT(Enumerate(set1) == MakeSet({&value})); + UNIT_ASSERT(Enumerate(set2) == MakeSet({&value})); + UNIT_ASSERT(Enumerate(set3) == MakeSet({&value})); + } + + Y_UNIT_TEST(SetAddedToManySets) { + TMyValue value; + TUnionCopySet set1, set2, set3; + set1.Add(&value); + set2.Add(set1); + set3.Add(set1); + UNIT_ASSERT(Enumerate(set1) == MakeSet({&value})); + UNIT_ASSERT(Enumerate(set2) == MakeSet({&value})); + UNIT_ASSERT(Enumerate(set3) == MakeSet({&value})); + } + + Y_UNIT_TEST(NotDisjoint) { + TVector values; + values.emplace_back(); + values.emplace_back(); + TUnionCopySet set; + set.Add(&values[0]); + set.Add(&values[0]); + set.Add(&values[0]); + set.Add(&values[1]); + set.Add(&values[1]); + set.Add(&values[1]); + UNIT_ASSERT_VALUES_EQUAL(Count(set), 6u); + values.pop_back(); + UNIT_ASSERT_VALUES_EQUAL(Count(set), 3u); + set.Clear(); + } + + Y_UNIT_TEST(NotDisjointOptimizeCopyPairWhileDestroying) { + TVector values; + values.emplace_back(); + TUnionCopySet set; + set.Add(&values[0]); + set.Add(&values[0]); + TUnionCopySet set1, set2, set3; + set1.Add(set); + set2.Add(set); + set3.Add(set); + // We should now have: (4 sets) <-> copy node <-> union node <-> (2 links) <-> copy node <-> value + // Destroying the value will trigger redundant union node optimization, + // which has links to a copy node pair, and where the smaller node is + // under destruction. + values.pop_back(); + } + + Y_UNIT_TEST(NotDisjointOptimizeUnionPairWhileDestroying) { + TVector values; + values.resize(4); + TUnionCopySet set; + set.Add(&values[0]); + set.Add(&values[1]); + set.Add(&values[2]); + set.Add(&values[3]); + // We have: set <-> union node <-> 4 values + // Making a copy produces: 2 sets <-> copy node <-> union node <-> 4 values + TUnionCopySet copy = set; + // When we add the set again a union node is added with a second link to union node + copy.Add(set); + // Clearning the original set will leave: + // copy <-> union node <-> (2 links) <-> copy node <-> union node <-> 4 values + set.Clear(); + // Clearning the copy will trigger redundant copy node optimization, + // which has links to a union node pair, and where the smaller node is + // under destruction. + copy.Clear(); + } + + Y_UNIT_TEST(MoveAdd) { + TVector values; + TUnionCopySet set1; + TUnionCopySet set2; + TUnionCopySet set3; + + // Adding empty set to an empty set + set1.Add(std::move(set2)); + UNIT_ASSERT_VALUES_EQUAL(Count(set1), 0u); + UNIT_ASSERT_VALUES_EQUAL(Count(set2), 0u); + + // Test adding existing values to an empty set + set2.Add(&values.emplace_back()); + set1.Add(std::move(set2)); + UNIT_ASSERT_VALUES_EQUAL(Count(set1), 1u); + UNIT_ASSERT_VALUES_EQUAL(Count(set2), 0u); + + // Testing when moving makes a set into a union + set3.Add(&values.emplace_back()); + set1.Add(std::move(set3)); + UNIT_ASSERT_VALUES_EQUAL(Count(set1), 2u); + UNIT_ASSERT_VALUES_EQUAL(Count(set3), 0u); + + // Test adding a union to a non-union + set2.Add(&values.emplace_back()); + set2.Add(std::move(set1)); + UNIT_ASSERT_VALUES_EQUAL(Count(set2), 3u); + UNIT_ASSERT_VALUES_EQUAL(Count(set1), 0u); + + // Testing adding union to a union + set1.Add(&values.emplace_back()); + set1.Add(&values.emplace_back()); + set1.Add(std::move(set2)); + UNIT_ASSERT_VALUES_EQUAL(Count(set1), 5u); + UNIT_ASSERT_VALUES_EQUAL(Count(set2), 0u); + } + + Y_UNIT_TEST(StressDestroyUp) { + // This n is enough to trigger stack overflow in release builds when implementation is recursive + size_t n = 1'000'000; + TVector items; + items.resize(n); + TVector> sets; + sets.resize(n); + TVector indexes; + indexes.reserve(n); + for (size_t i = 0; i < n; ++i) { + indexes.push_back(i); + } + ShuffleRange(indexes); + for (size_t i = 0; i < n; ++i) { + if (i > 0) { + sets[i].Add(sets[i-1]); + } + sets[i].Add(&items[indexes[i]]); + } + // The last set must contain all values + UNIT_ASSERT_VALUES_EQUAL(Count(sets.back()), n); + while (!items.empty()) { + items.pop_back(); + } + } + + Y_UNIT_TEST(StressDestroyDown) { + // This n is enough to trigger stack overflow in release builds when implementation is recursive + size_t n = 1'000'000; + TVector items; + items.resize(n); + TVector> sets; + sets.resize(n); + for (size_t i = 0; i < n; ++i) { + sets[i].Add(&items[i]); + if (i > 0) { + sets[i].Add(sets[i-1]); + } + } + // The last set must contain all values + UNIT_ASSERT_VALUES_EQUAL(Count(sets.back()), n); + TVector indexes; + indexes.reserve(n); + for (size_t i = 0; i < n; ++i) { + indexes.push_back(i); + } + ShuffleRange(indexes); + for (size_t i : indexes) { + sets[i].Clear(); + } + } + + Y_UNIT_TEST(MovingSetsAndValues) { + TVector values; + TVector> sets; + for (size_t i = 0; i < 1024; ++i) { + // Sets and values will be move constructed when vector capacity changes + sets.emplace_back().Add(&values.emplace_back()); + // Make sets and values more complex after some simple moves + if (i >= 128) { + sets.back().Add(sets[i - 128]); + } + } + } + + Y_UNIT_TEST(DeepDestroyUp) { + // This n is enough to trigger stack overflow in release builds when implementation is recursive + size_t n = 1'000'000; + std::optional value; + TUnionCopySet set; + set.Add(&value.emplace()); + for (size_t i = 0; i < n; ++i) { + // Repeatedly add set to itself via a copy + // This will create a chain of copy and union nodes with 2 links between each + auto copy = set; + set.Add(copy); + } + // Note: there are currently 2^n duplicate elements in set! + // Destroy the value, which will unlink and destroy everything + value.reset(); + } + + Y_UNIT_TEST(DeepDestroyDown) { + // This n is enough to trigger stack overflow in release builds when implementation is recursive + size_t n = 1'000'000; + TMyValue value; + TUnionCopySet set; + set.Add(&value); + for (size_t i = 0; i < n; ++i) { + // Repeatedly add set to itself via a copy + // This will create a chain of copy and union nodes with 2 links between each + auto copy = set; + set.Add(copy); + } + // Note: there are currently 2^n duplicate elements in set! + // Clear the set, which will unlink and destroy everything + set.Clear(); + } + + Y_UNIT_TEST(DebugString) { + TMyValue a(1), b(2), c(3); + TUnionCopySet set1; + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "empty"); + set1.Add(&a); + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "1"); + set1.Add(&b); + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "Union{ 1, 2 }"); + TUnionCopySet set2; + set2.Add(&a); + UNIT_ASSERT_VALUES_EQUAL(set2.DebugString(), "Copy -> 1"); + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "Union{ Copy -> 1, 2 }"); + TUnionCopySet set3 = set1; + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "Copy -> Union{ Copy -> 1, 2 }"); + UNIT_ASSERT_VALUES_EQUAL(set3.DebugString(), "Copy -> Union{ Copy -> 1, 2 }"); + set2.Clear(); + set3.Clear(); + UNIT_ASSERT_VALUES_EQUAL(set1.DebugString(), "Union{ 1, 2 }"); + } +} + +} // namespace NKikimr diff --git a/ydb/library/union_copy_set/ut/ya.make b/ydb/library/union_copy_set/ut/ya.make new file mode 100644 index 000000000000..21fedf176d56 --- /dev/null +++ b/ydb/library/union_copy_set/ut/ya.make @@ -0,0 +1,9 @@ +UNITTEST_FOR(ydb/library/union_copy_set) + +SIZE(MEDIUM) + +SRCS( + union_copy_set_ut.cpp +) + +END() diff --git a/ydb/library/union_copy_set/ya.make b/ydb/library/union_copy_set/ya.make new file mode 100644 index 000000000000..06b68b379d62 --- /dev/null +++ b/ydb/library/union_copy_set/ya.make @@ -0,0 +1,15 @@ +LIBRARY() + +PEERDIR( + library/cpp/containers/stack_vector +) + +SRCS( + union_copy_set.h +) + +END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/ya.make b/ydb/library/ya.make index 95aea84ff6fa..bfada38eae66 100644 --- a/ydb/library/ya.make +++ b/ydb/library/ya.make @@ -15,6 +15,7 @@ RECURSE( global_plugins grpc http_proxy + intersection_tree keys lockfree_bucket logger @@ -35,6 +36,7 @@ RECURSE( table_creator testlib time_series_vec + union_copy_set wilson_ids workload workload From 27d464e0afbe9f7ef7fa1d984fc94ff4e342fb81 Mon Sep 17 00:00:00 2001 From: Aleksei Borzenkov Date: Wed, 11 Jun 2025 15:22:36 +0300 Subject: [PATCH 56/62] Use Y_ENSURE for usage errors in intersection tree (#19590) --- ydb/library/intersection_tree/intersection_tree.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/ydb/library/intersection_tree/intersection_tree.h b/ydb/library/intersection_tree/intersection_tree.h index e85d700f7e57..4758507bb4c1 100644 --- a/ydb/library/intersection_tree/intersection_tree.h +++ b/ydb/library/intersection_tree/intersection_tree.h @@ -194,7 +194,7 @@ namespace NKikimr { std::piecewise_construct, std::forward_as_tuple(value), std::forward_as_tuple(value, leftKey, rightKey)); - Y_ABORT_UNLESS(res.second); + Y_ENSURE(res.second); TValueNode* valueNode = &res.first->second; TNode* left = FindOrInsert(valueNode->LeftKey, RandomNumber()); @@ -213,7 +213,7 @@ namespace NKikimr { TNode* left = Find(valueNode->LeftKey); TNode* right = Find(valueNode->RightKey); - Y_ABORT_UNLESS(left && right); + Y_ENSURE(left && right); left->UseCount--; right->UseCount--; @@ -269,7 +269,7 @@ namespace NKikimr { } const TPartitionKey& GetLeftPartitionKey() const { - Y_ABORT_UNLESS(LeftBorder); + Y_ENSURE(LeftBorder); return LeftBorder->Key; } @@ -286,7 +286,7 @@ namespace NKikimr { } const TPartitionKey& GetRightPartitionKey() const { - Y_ABORT_UNLESS(RightBorder); + Y_ENSURE(RightBorder); return RightBorder->Key; } @@ -372,7 +372,7 @@ namespace NKikimr { count -= parent->RightDelta; return {parent, count}; } - Y_ABORT_UNLESS(parent->Left.get() == node); + Y_DEBUG_ABORT_UNLESS(parent->Left.get() == node); count -= parent->RightDelta; node = parent; } @@ -395,7 +395,7 @@ namespace NKikimr { count -= parent->LeftDelta; return {parent, count}; } - Y_ABORT_UNLESS(parent->Right.get() == node); + Y_DEBUG_ABORT_UNLESS(parent->Right.get() == node); count -= parent->RightDelta; node = parent; } @@ -574,7 +574,7 @@ namespace NKikimr { while (p != q) { p = p->Parent; q = q->Parent; - Y_ABORT_UNLESS(p || q); + Y_DEBUG_ABORT_UNLESS(p || q); if (!p) { p = b; } @@ -719,7 +719,6 @@ namespace NKikimr { *tptr = Merge(d->RemoveLeft(), d->RemoveRight()); tptr->get()->Parent = parent; } else if (parent) { - // Y_ABORT_UNLESS(d->LeftDelta == d->RightDelta); if (leftChild) { parent->LeftDelta += d->LeftDelta; parent->LeftValues += std::move(d->LeftValues); From 7186bd1ce86dacf89181e61d47cc438d4a748300 Mon Sep 17 00:00:00 2001 From: ivanmorozov333 Date: Thu, 26 Jun 2025 12:07:24 +0300 Subject: [PATCH 57/62] memory distribution control (#20173) Co-authored-by: ivanmorozov333 --- .../grouped_memory/service/manager.cpp | 57 ++++++++++++++----- .../limiter/grouped_memory/service/manager.h | 40 +++++++++++-- .../grouped_memory/service/process.cpp | 6 ++ .../limiter/grouped_memory/service/process.h | 57 +++++++++++++++++-- 4 files changed, 137 insertions(+), 23 deletions(-) diff --git a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp index 96fe8bcefc17..3584e8612bc9 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp @@ -25,6 +25,7 @@ void TManager::UnregisterGroup(const ui64 externalProcessId, const ui64 external AFL_DEBUG(NKikimrServices::GROUPED_MEMORY_LIMITER)("event", "unregister_group")("external_process_id", externalProcessId)( "external_group_id", externalGroupId)("size", ProcessIds.GetSize()); if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { + auto g = BuildProcessOrderGuard(*process); process->UnregisterGroup(externalScopeId, externalGroupId); } RefreshSignals(); @@ -32,7 +33,16 @@ void TManager::UnregisterGroup(const ui64 externalProcessId, const ui64 external void TManager::UpdateAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId, const ui64 volume) { TProcessMemory& process = GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)); - if (process.UpdateAllocation(externalScopeId, allocationId, volume)) { + bool updated = false; + { + auto g = BuildProcessOrderGuard(process); + updated = process.UpdateAllocation(externalScopeId, allocationId, volume); + if (!updated) { + g.Release(); + } + } + + if (updated) { TryAllocateWaiting(); } @@ -46,15 +56,18 @@ void TManager::TryAllocateWaiting() { AFL_VERIFY(it->second.IsPriorityProcess()); it->second.TryAllocateWaiting(0); } - while (true) { - bool found = false; - for (auto&& i : Processes) { - if (i.second.TryAllocateWaiting(1)) { - found = true; + for (auto it = ProcessesOrdered.begin(); it != ProcessesOrdered.end();) { + if (it->second->TryAllocateWaiting(1)) { + TProcessMemory* process = it->second; + it = ProcessesOrdered.erase(it); + auto info = ProcessesOrdered.emplace(process->BuildUsageAddress(), process); + AFL_VERIFY(info.second); + auto itNew = info.first; + if (it == ProcessesOrdered.end() || itNew->first < it->first) { + it = itNew; } - } - if (!found) { - break; + } else { + ++it; } } RefreshSignals(); @@ -62,7 +75,15 @@ void TManager::TryAllocateWaiting() { void TManager::UnregisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId) { if (auto* process = GetProcessMemoryByExternalIdOptional(externalProcessId)) { - if (process->UnregisterAllocation(externalScopeId, allocationId)) { + bool unregistered = false; + { + auto g = BuildProcessOrderGuard(*process); + unregistered = process->UnregisterAllocation(externalScopeId, allocationId); + if (!unregistered) { + g.Release(); + } + } + if (unregistered) { TryAllocateWaiting(); } } @@ -75,7 +96,7 @@ void TManager::RegisterAllocation(const ui64 externalProcessId, const ui64 exter process->RegisterAllocation(externalScopeId, externalGroupId, task, stageIdx); } else { AFL_VERIFY(!task->OnAllocated(std::make_shared(externalProcessId, externalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( - "ext_group", externalGroupId)("stage_idx", stageIdx); + "ext_group", externalGroupId)("stage_idx", stageIdx); } RefreshSignals(); } @@ -84,7 +105,10 @@ void TManager::RegisterProcess(const ui64 externalProcessId, const std::vectorsecond.BuildUsageAddress(), &info.first->second); } else { ++Processes.find(*internalId)->second.MutableLinksCount(); } @@ -99,6 +123,7 @@ void TManager::UnregisterProcess(const ui64 externalProcessId) { return; } Y_UNUSED(ProcessIds.ExtractInternalIdVerified(externalProcessId)); + AFL_VERIFY(ProcessesOrdered.erase(it->second.BuildUsageAddress())); it->second.Unregister(); Processes.erase(it); const ui64 nextInternalProcessId = ProcessIds.GetMinInternalIdDef(internalProcessId); @@ -110,12 +135,16 @@ void TManager::UnregisterProcess(const ui64 externalProcessId) { } void TManager::RegisterProcessScope(const ui64 externalProcessId, const ui64 externalProcessScopeId) { - GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)).RegisterScope(externalProcessScopeId); + auto& process = GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)); + auto g = BuildProcessOrderGuard(process); + process.RegisterScope(externalProcessScopeId); RefreshSignals(); } void TManager::UnregisterProcessScope(const ui64 externalProcessId, const ui64 externalProcessScopeId) { - GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)).UnregisterScope(externalProcessScopeId); + auto& process = GetProcessMemoryVerified(ProcessIds.GetInternalIdVerified(externalProcessId)); + auto g = BuildProcessOrderGuard(process); + process.UnregisterScope(externalProcessScopeId); RefreshSignals(); } diff --git a/ydb/core/tx/limiter/grouped_memory/service/manager.h b/ydb/core/tx/limiter/grouped_memory/service/manager.h index fd641a3f69b1..4fec258a3f00 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/manager.h +++ b/ydb/core/tx/limiter/grouped_memory/service/manager.h @@ -20,6 +20,7 @@ class TManager { const std::shared_ptr Signals; const NActors::TActorId OwnerActorId; THashMap Processes; + std::map ProcessesOrdered; std::shared_ptr DefaultStage; TIdsControl ProcessIds; @@ -28,6 +29,39 @@ class TManager { Signals->ProcessesCount->Set(Processes.size()); } + class TOrderedProcessesGuard { + private: + bool Released = false; + TProcessMemory& Process; + std::map* Processes; + TProcessMemoryUsage Start; + + public: + TOrderedProcessesGuard(TProcessMemory& process, std::map& processes) + : Process(process) + , Processes(&processes) + , Start(Process.BuildUsageAddress()) { + AFL_VERIFY(Processes->contains(Start)); + } + + ~TOrderedProcessesGuard() { + if (Released) { + return; + } + AFL_VERIFY(Processes->erase(Start))("start", Start.DebugString()); + AFL_VERIFY(Processes->emplace(Process.BuildUsageAddress(), &Process).second); + } + + void Release() { + AFL_VERIFY(!Released); + Released = true; + } + }; + + TOrderedProcessesGuard BuildProcessOrderGuard(TProcessMemory& process) { + return TOrderedProcessesGuard(process, ProcessesOrdered); + } + TProcessMemory& GetProcessMemoryVerified(const ui64 internalProcessId) { auto it = Processes.find(internalProcessId); AFL_VERIFY(it != Processes.end()); @@ -52,8 +86,7 @@ class TManager { , Name(name) , Signals(signals) , OwnerActorId(ownerActorId) - , DefaultStage(defaultStage) - { + , DefaultStage(defaultStage) { } void RegisterGroup(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId); @@ -66,8 +99,7 @@ class TManager { void UnregisterProcess(const ui64 externalProcessId); void RegisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 externalGroupId, - const std::shared_ptr& task, - const std::optional& stageIdx); + const std::shared_ptr& task, const std::optional& stageIdx); void UnregisterAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId); void UpdateAllocation(const ui64 externalProcessId, const ui64 externalScopeId, const ui64 allocationId, const ui64 volume); diff --git a/ydb/core/tx/limiter/grouped_memory/service/process.cpp b/ydb/core/tx/limiter/grouped_memory/service/process.cpp index bcde6532e797..03a2e4247e2e 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/process.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/process.cpp @@ -1,5 +1,11 @@ #include "process.h" +#include + namespace NKikimr::NOlap::NGroupedMemoryManager { +TString TProcessMemoryUsage::DebugString() const { + return TStringBuilder() << "{mem:" << MemoryUsage << ";process_id:" << InternalProcessId << "}"; +} + } // namespace NKikimr::NOlap::NGroupedMemoryManager diff --git a/ydb/core/tx/limiter/grouped_memory/service/process.h b/ydb/core/tx/limiter/grouped_memory/service/process.h index 524c40566576..f852a14c9b20 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/process.h +++ b/ydb/core/tx/limiter/grouped_memory/service/process.h @@ -2,9 +2,8 @@ #include "group.h" #include "ids.h" -#include - #include +#include namespace NKikimr::NOlap::NGroupedMemoryManager { @@ -152,12 +151,32 @@ class TProcessMemoryScope: public NColumnShard::TMonitoringObjectsCounter { private: const ui64 ExternalProcessId; + const ui64 InternalProcessId; const NActors::TActorId OwnerActorId; bool PriorityProcessFlag = false; + ui64 MemoryUsage = 0; YDB_ACCESSOR(ui32, LinksCount, 1); YDB_READONLY_DEF(std::vector>, Stages); @@ -176,13 +195,30 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCounterGetUsage().Val(); + } + MemoryUsage = result; + } + public: + TProcessMemoryUsage BuildUsageAddress() const { + return TProcessMemoryUsage(MemoryUsage, InternalProcessId); + } + bool IsPriorityProcess() const { return PriorityProcessFlag; } bool UpdateAllocation(const ui64 externalScopeId, const ui64 allocationId, const ui64 volume) { - return GetAllocationScopeVerified(externalScopeId).UpdateAllocation(allocationId, volume); + if (GetAllocationScopeVerified(externalScopeId).UpdateAllocation(allocationId, volume)) { + RefreshMemoryUsage(); + return true; + } else { + return false; + } } void RegisterAllocation( @@ -204,7 +240,10 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCounterUnregisterAllocation(allocationId); + if (scope->UnregisterAllocation(allocationId)) { + RefreshMemoryUsage(); + return true; + } } return false; } @@ -212,6 +251,7 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCounterUnregisterGroup(IsPriorityProcess(), externalGroupId); + RefreshMemoryUsage(); } } @@ -224,6 +264,7 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCountersecond->Unregister()) { AllocationScopes.erase(it); + RefreshMemoryUsage(); } } @@ -241,9 +282,10 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCounter>& stages, const std::shared_ptr& defaultStage) : ExternalProcessId(externalProcessId) + , InternalProcessId(internalProcessId) , OwnerActorId(ownerActorId) , PriorityProcessFlag(isPriority) , Stages(stages) @@ -257,6 +299,9 @@ class TProcessMemory: public NColumnShard::TMonitoringObjectsCounterUnregister()); } + RefreshMemoryUsage(); +// AFL_VERIFY(MemoryUsage == 0)("usage", MemoryUsage); AllocationScopes.clear(); } }; From 1196e82687ffaaa7a8755ef3d809fc91ab19acd0 Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Tue, 8 Jul 2025 07:48:57 +0000 Subject: [PATCH 58/62] unused mark --- ydb/core/tx/conveyor_composite/usage/common.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/tx/conveyor_composite/usage/common.cpp b/ydb/core/tx/conveyor_composite/usage/common.cpp index b1090664ed32..dfa6ad86169c 100644 --- a/ydb/core/tx/conveyor_composite/usage/common.cpp +++ b/ydb/core/tx/conveyor_composite/usage/common.cpp @@ -22,6 +22,7 @@ TProcessGuard::TProcessGuard(const ESpecialTaskCategory category, const TString& , ScopeId(scopeId) , ExternalProcessId(externalProcessId) , ServiceActorId(actorId) { + Y_UNUSED(ExternalProcessId); if (ServiceActorId) { NActors::TActorContext::AsActorContext().Send( *ServiceActorId, new NConveyorComposite::TEvExecution::TEvRegisterProcess(cpuLimits, category, scopeId, InternalProcessId)); From 25f5f1db478677bceb51674854edf88c39322b5d Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Tue, 8 Jul 2025 09:22:29 +0000 Subject: [PATCH 59/62] cleanup --- ydb/core/kqp/ut/olap/sparsed_ut.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/kqp/ut/olap/sparsed_ut.cpp b/ydb/core/kqp/ut/olap/sparsed_ut.cpp index 8da029edd9fa..e5cc03bcb758 100644 --- a/ydb/core/kqp/ut/olap/sparsed_ut.cpp +++ b/ydb/core/kqp/ut/olap/sparsed_ut.cpp @@ -258,7 +258,6 @@ Y_UNIT_TEST_SUITE(KqpOlapSparsed) { } void ExecuteMultiColumn() { - CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Indexation); CSController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::Compaction); CSController->SetOverridePeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100)); From 1d4f6fb5e1929b269655b652ffa38b15e3b50e23 Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Tue, 8 Jul 2025 10:58:59 +0000 Subject: [PATCH 60/62] is stopped --- ydb/library/actors/core/actorsystem.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ydb/library/actors/core/actorsystem.cpp b/ydb/library/actors/core/actorsystem.cpp index f964154dda53..8d2f070d7672 100644 --- a/ydb/library/actors/core/actorsystem.cpp +++ b/ydb/library/actors/core/actorsystem.cpp @@ -92,6 +92,13 @@ namespace NActors { Cleanup(); } + bool TActorSystem::IsStopped() { + if (!TlsActivationContext) { + return true; + } + return TlsActivationContext->ActorSystem()->StopExecuted || !TlsActivationContext->ActorSystem()->StartExecuted; + } + template bool TActorSystem::GenericSend(TAutoPtr ev) const { if (Y_UNLIKELY(!ev)) From d4f33a3503383ece588a3188b1e630d6d9d01622 Mon Sep 17 00:00:00 2001 From: Pisarenko Grigoriy Date: Thu, 10 Apr 2025 15:15:14 +0300 Subject: [PATCH 61/62] YQ-4222 added nested protos validation (#16975) --- ydb/core/fq/libs/actors/pending_fetcher.cpp | 4 +- ydb/core/fq/libs/actors/proxy.h | 1 - ydb/core/fq/libs/actors/run_actor.cpp | 16 +++-- .../libs/actors/ut/database_resolver_ut.cpp | 6 +- ydb/core/fq/libs/actors/ya.make | 3 +- .../database_resolver.cpp | 0 .../database_resolver.h | 0 .../http_proxy.cpp} | 4 +- .../db_id_async_resolver_impl/http_proxy.h | 9 +++ .../fq/libs/db_id_async_resolver_impl/ya.make | 11 +++ ydb/core/fq/libs/init/init.cpp | 1 + ydb/core/fq/libs/init/ya.make | 1 + ydb/core/fq/libs/mock/ya.make | 1 + ydb/core/fq/libs/mock/yql_mock.cpp | 1 + .../libs/test_connection/test_connection.cpp | 3 +- .../kqp_federated_query_helpers.cpp | 59 +++++++++++++++- .../kqp_federated_query_helpers.h | 17 +++++ .../kqp_federated_query_helpers_ut.cpp | 68 +++++++++++++++++++ ydb/core/kqp/federated_query/ut/ya.make | 20 ++++++ ydb/core/kqp/federated_query/ya.make | 6 ++ .../run_script_actor/kqp_run_script_actor.cpp | 18 ++++- ydb/core/testlib/test_client.cpp | 3 +- ydb/services/fq/ut_integration/fq_ut.cpp | 2 +- ydb/tests/fq/s3/test_empty.py | 21 ------ ydb/tests/fq/s3/test_validation.py | 59 ++++++++++++++++ ydb/tests/fq/s3/ya.make | 2 +- 26 files changed, 291 insertions(+), 45 deletions(-) rename ydb/core/fq/libs/{actors => db_id_async_resolver_impl}/database_resolver.cpp (100%) rename ydb/core/fq/libs/{actors => db_id_async_resolver_impl}/database_resolver.h (100%) rename ydb/core/fq/libs/{actors/proxy.cpp => db_id_async_resolver_impl/http_proxy.cpp} (79%) create mode 100644 ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.h create mode 100644 ydb/core/kqp/federated_query/kqp_federated_query_helpers_ut.cpp create mode 100644 ydb/core/kqp/federated_query/ut/ya.make delete mode 100644 ydb/tests/fq/s3/test_empty.py create mode 100644 ydb/tests/fq/s3/test_validation.py diff --git a/ydb/core/fq/libs/actors/pending_fetcher.cpp b/ydb/core/fq/libs/actors/pending_fetcher.cpp index 3f2db94b8351..42789883052d 100644 --- a/ydb/core/fq/libs/actors/pending_fetcher.cpp +++ b/ydb/core/fq/libs/actors/pending_fetcher.cpp @@ -1,8 +1,6 @@ #include "proxy.h" #include "nodes_manager.h" -#include "database_resolver.h" - #include #include #include @@ -52,6 +50,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/ydb/core/fq/libs/actors/proxy.h b/ydb/core/fq/libs/actors/proxy.h index 9f457859189b..d86dd3cd5bbe 100644 --- a/ydb/core/fq/libs/actors/proxy.h +++ b/ydb/core/fq/libs/actors/proxy.h @@ -35,7 +35,6 @@ namespace NKikimr { namespace NFq { -NActors::TActorId MakeYqlAnalyticsHttpProxyId(); NActors::TActorId MakePendingFetcherId(ui32 nodeId); NActors::IActor* CreatePendingFetcher( diff --git a/ydb/core/fq/libs/actors/run_actor.cpp b/ydb/core/fq/libs/actors/run_actor.cpp index a63757261021..548276bf7cd5 100644 --- a/ydb/core/fq/libs/actors/run_actor.cpp +++ b/ydb/core/fq/libs/actors/run_actor.cpp @@ -67,6 +67,7 @@ #include #include #include +#include #include #include @@ -957,6 +958,7 @@ class TRunActor : public NActors::TActorBootstrapped { Fq::Private::PingTaskRequest request; if (proto.issues_size()) { *request.mutable_transient_issues() = proto.issues(); + NKikimr::NKqp::TruncateIssues(request.mutable_transient_issues()); } if (proto.metric_size()) { TString statistics; @@ -971,7 +973,7 @@ class TRunActor : public NActors::TActorBootstrapped { void SendTransientIssues(const NYql::TIssues& issues) { Fq::Private::PingTaskRequest request; - NYql::IssuesToMessage(issues, request.mutable_transient_issues()); + NYql::IssuesToMessage(NKikimr::NKqp::TruncateIssues(issues), request.mutable_transient_issues()); Send(Pinger, new TEvents::TEvForwardPingRequest(request), 0, RaiseTransientIssuesCookie); } @@ -1017,6 +1019,12 @@ class TRunActor : public NActors::TActorBootstrapped { const auto emptyResultSet = builder.BuildResultSet({}); auto* header = QueryStateUpdateRequest.add_result_set_meta(); (*header->mutable_column()) = emptyResultSet.columns(); + + if (const auto& issues = NKikimr::NKqp::ValidateResultSetColumns(header->column())) { + header->clear_column(); + Abort("Invalid result set columns, please contact internal support", FederatedQuery::QueryMeta::FAILED, issues); + return; + } } } *request.mutable_result_set_meta() = QueryStateUpdateRequest.result_set_meta(); @@ -1190,7 +1198,7 @@ class TRunActor : public NActors::TActorBootstrapped { } TIssue WrapInternalIssues(const TIssues& issues) { - NYql::IssuesToMessage(issues, QueryStateUpdateRequest.mutable_internal_issues()); + NYql::IssuesToMessage(NKikimr::NKqp::TruncateIssues(issues), QueryStateUpdateRequest.mutable_internal_issues()); TString referenceId = GetEntityIdAsString(Params.Config.GetCommon().GetIdsPrefix(), EEntityType::UNDEFINED); LOG_E(referenceId << ": " << issues.ToOneLineString()); return TIssue("Contact technical support and provide query information and this id: " + referenceId + "_" + Now().ToStringUpToSeconds()); @@ -1876,8 +1884,8 @@ class TRunActor : public NActors::TActorBootstrapped { Issues.Clear(); } - NYql::IssuesToMessage(TransientIssues, QueryStateUpdateRequest.mutable_transient_issues()); - NYql::IssuesToMessage(Issues, QueryStateUpdateRequest.mutable_issues()); + NYql::IssuesToMessage(NKikimr::NKqp::TruncateIssues(TransientIssues), QueryStateUpdateRequest.mutable_transient_issues()); + NYql::IssuesToMessage(NKikimr::NKqp::TruncateIssues(Issues), QueryStateUpdateRequest.mutable_issues()); /* 1. If the execution has already started then the issue will be put through TEvAbortExecution 2. If execution hasn't started then the issue will be put in this place diff --git a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp index 59a4cb47ae07..d9a81685a2dd 100644 --- a/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp +++ b/ydb/core/fq/libs/actors/ut/database_resolver_ut.cpp @@ -1,7 +1,7 @@ -#include -#include -#include #include +#include +#include +#include #include #include diff --git a/ydb/core/fq/libs/actors/ya.make b/ydb/core/fq/libs/actors/ya.make index 80da96cdb259..8379bbc4e7d0 100644 --- a/ydb/core/fq/libs/actors/ya.make +++ b/ydb/core/fq/libs/actors/ya.make @@ -2,12 +2,10 @@ LIBRARY() SRCS( clusters_from_connections.cpp - database_resolver.cpp error.cpp nodes_health_check.cpp nodes_manager.cpp pending_fetcher.cpp - proxy.cpp proxy_private.cpp rate_limiter.cpp rate_limiter_resources.cpp @@ -50,6 +48,7 @@ PEERDIR( ydb/core/fq/libs/result_formatter ydb/core/fq/libs/shared_resources ydb/core/fq/libs/signer + ydb/core/kqp/federated_query ydb/core/protos ydb/core/util ydb/library/mkql_proto diff --git a/ydb/core/fq/libs/actors/database_resolver.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/database_resolver.cpp similarity index 100% rename from ydb/core/fq/libs/actors/database_resolver.cpp rename to ydb/core/fq/libs/db_id_async_resolver_impl/database_resolver.cpp diff --git a/ydb/core/fq/libs/actors/database_resolver.h b/ydb/core/fq/libs/db_id_async_resolver_impl/database_resolver.h similarity index 100% rename from ydb/core/fq/libs/actors/database_resolver.h rename to ydb/core/fq/libs/db_id_async_resolver_impl/database_resolver.h diff --git a/ydb/core/fq/libs/actors/proxy.cpp b/ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.cpp similarity index 79% rename from ydb/core/fq/libs/actors/proxy.cpp rename to ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.cpp index ca6da139c00d..ab49bd86f595 100644 --- a/ydb/core/fq/libs/actors/proxy.cpp +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.cpp @@ -1,9 +1,7 @@ -#include "proxy.h" +#include "http_proxy.h" namespace NFq { -using namespace NActors; - NActors::TActorId MakeYqlAnalyticsHttpProxyId() { constexpr TStringBuf name = "YQLHTTPROXY"; return NActors::TActorId(0, name); diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.h b/ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.h new file mode 100644 index 000000000000..ab713401cd29 --- /dev/null +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/http_proxy.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace NFq { + +NActors::TActorId MakeYqlAnalyticsHttpProxyId(); + +} // namespace NFq diff --git a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make index 97b9ea6f273b..7a3bf5dbf79a 100644 --- a/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make +++ b/ydb/core/fq/libs/db_id_async_resolver_impl/ya.make @@ -1,15 +1,26 @@ LIBRARY() SRCS( + database_resolver.cpp db_async_resolver_impl.cpp + http_proxy.cpp mdb_endpoint_generator.cpp ) PEERDIR( + library/cpp/json library/cpp/threading/future + ydb/core/fq/libs/common + ydb/core/fq/libs/config/protos ydb/core/fq/libs/events + ydb/core/util + ydb/library/actors/core + ydb/library/actors/http + ydb/library/services ydb/library/yql/providers/common/db_id_async_resolver + ydb/library/yql/providers/common/token_accessor/client ydb/library/yql/providers/dq/actors + yql/essentials/utils ) YQL_LAST_ABI_VERSION() diff --git a/ydb/core/fq/libs/init/init.cpp b/ydb/core/fq/libs/init/init.cpp index db4cda85d81c..2f074c5e9530 100644 --- a/ydb/core/fq/libs/init/init.cpp +++ b/ydb/core/fq/libs/init/init.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/ydb/core/fq/libs/init/ya.make b/ydb/core/fq/libs/init/ya.make index c4ad1e08b544..caaeb50b8b43 100644 --- a/ydb/core/fq/libs/init/ya.make +++ b/ydb/core/fq/libs/init/ya.make @@ -16,6 +16,7 @@ PEERDIR( ydb/core/fq/libs/control_plane_config ydb/core/fq/libs/control_plane_proxy ydb/core/fq/libs/control_plane_storage + ydb/core/fq/libs/db_id_async_resolver_impl ydb/core/fq/libs/events ydb/core/fq/libs/gateway ydb/core/fq/libs/health diff --git a/ydb/core/fq/libs/mock/ya.make b/ydb/core/fq/libs/mock/ya.make index 219773f916e7..9d918a5fac36 100644 --- a/ydb/core/fq/libs/mock/ya.make +++ b/ydb/core/fq/libs/mock/ya.make @@ -15,6 +15,7 @@ PEERDIR( ydb/core/base ydb/core/fq/libs/actors ydb/core/fq/libs/common + ydb/core/fq/libs/db_id_async_resolver_impl ydb/core/fq/libs/db_schema ydb/core/fq/libs/shared_resources/interface ydb/core/protos diff --git a/ydb/core/fq/libs/mock/yql_mock.cpp b/ydb/core/fq/libs/mock/yql_mock.cpp index 50b8e8af6244..60f4bcc22e41 100644 --- a/ydb/core/fq/libs/mock/yql_mock.cpp +++ b/ydb/core/fq/libs/mock/yql_mock.cpp @@ -2,6 +2,7 @@ #include #include +#include #include diff --git a/ydb/core/fq/libs/test_connection/test_connection.cpp b/ydb/core/fq/libs/test_connection/test_connection.cpp index 89993c8fdaa9..92739ad0ea80 100644 --- a/ydb/core/fq/libs/test_connection/test_connection.cpp +++ b/ydb/core/fq/libs/test_connection/test_connection.cpp @@ -4,11 +4,12 @@ #include "request_validators.h" #include -#include #include #include #include +#include #include +#include #include #include diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp index 26d7e7cda7d1..64ac377e9c37 100644 --- a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp +++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.cpp @@ -7,11 +7,13 @@ #include #include -#include -#include +#include #include +#include #include +#include + #include #include #include @@ -22,6 +24,35 @@ #include namespace NKikimr::NKqp { + + bool CheckNestingDepth(const google::protobuf::Message& message, ui32 maxDepth) { + if (!maxDepth) { + return false; + } + --maxDepth; + + const auto* descriptor = message.GetDescriptor(); + const auto* reflection = message.GetReflection(); + for (int i = 0; i < descriptor->field_count(); ++i) { + const auto* field = descriptor->field(i); + if (field->cpp_type() != google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + continue; + } + + if (field->is_repeated()) { + for (int j = 0; j < reflection->FieldSize(message, field); ++j) { + if (!CheckNestingDepth(reflection->GetRepeatedMessage(message, field, j), maxDepth)) { + return false; + } + } + } else if (reflection->HasField(message, field) && !CheckNestingDepth(reflection->GetMessage(message, field), maxDepth)) { + return false; + } + } + + return true; + } + NYql::IYtGateway::TPtr MakeYtGateway(const NMiniKQL::IFunctionRegistry* functionRegistry, const NKikimrConfig::TQueryServiceConfig& queryServiceConfig) { NYql::TYtNativeServices ytServices; ytServices.FunctionRegistry = functionRegistry; @@ -212,4 +243,28 @@ namespace NKikimr::NKqp { return false; } + + NYql::TIssues TruncateIssues(const NYql::TIssues& issues, ui32 maxLevels, ui32 keepTailLevels) { + const auto options = NYql::TTruncateIssueOpts() + .SetMaxLevels(maxLevels) + .SetKeepTailLevels(keepTailLevels); + + NYql::TIssues result; + result.Reserve(issues.Size()); + for (const auto& issue : issues) { + result.AddIssue(NYql::TruncateIssueLevels(issue, options)); + } + return result; + } + + NYql::TIssues ValidateResultSetColumns(const google::protobuf::RepeatedPtrField& columns, ui32 maxNestingDepth) { + NYql::TIssues issues; + for (const auto& column : columns) { + if (!CheckNestingDepth(column.type(), maxNestingDepth)) { + issues.AddIssue(NYql::TIssue(TStringBuilder() << "Nesting depth of type for result column '" << column.name() << "' large than allowed limit " << maxNestingDepth)); + } + } + return issues; + } + } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.h b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.h index ca3478ff152b..7905a804e1ed 100644 --- a/ydb/core/kqp/federated_query/kqp_federated_query_helpers.h +++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers.h @@ -11,6 +11,8 @@ #include #include #include +#include + #include namespace NKikimrConfig { @@ -18,6 +20,9 @@ namespace NKikimrConfig { } namespace NKikimr::NKqp { + + bool CheckNestingDepth(const google::protobuf::Message& message, ui32 maxDepth); + NYql::IYtGateway::TPtr MakeYtGateway(const NMiniKQL::IFunctionRegistry* functionRegistry, const NKikimrConfig::TQueryServiceConfig& queryServiceConfig); NYql::IHTTPGateway::TPtr MakeHttpGateway(const NYql::THttpGatewayConfig& httpGatewayConfig, NMonitoring::TDynamicCounterPtr countersRoot); @@ -133,4 +138,16 @@ namespace NKikimr::NKqp { // Used only for unit tests bool WaitHttpGatewayFinalization(NMonitoring::TDynamicCounterPtr countersRoot, TDuration timeout = TDuration::Minutes(1), TDuration refreshPeriod = TDuration::MilliSeconds(100)); + + NYql::TIssues TruncateIssues(const NYql::TIssues& issues, ui32 maxLevels = 50, ui32 keepTailLevels = 3); + + template + void TruncateIssues(google::protobuf::RepeatedPtrField* issuesProto, ui32 maxLevels = 50, ui32 keepTailLevels = 3) { + NYql::TIssues issues; + NYql::IssuesFromMessage(*issuesProto, issues); + NYql::IssuesToMessage(TruncateIssues(issues, maxLevels, keepTailLevels), issuesProto); + } + + NYql::TIssues ValidateResultSetColumns(const google::protobuf::RepeatedPtrField& columns, ui32 maxNestingDepth = 90); + } // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/federated_query/kqp_federated_query_helpers_ut.cpp b/ydb/core/kqp/federated_query/kqp_federated_query_helpers_ut.cpp new file mode 100644 index 000000000000..f5996b0b3410 --- /dev/null +++ b/ydb/core/kqp/federated_query/kqp_federated_query_helpers_ut.cpp @@ -0,0 +1,68 @@ +#include "kqp_federated_query_helpers.h" + +#include + +#include + +#include + +namespace NKikimr::NKqp { + +Y_UNIT_TEST_SUITE(TestFederatedQueryHelpers) { + NYql::TIssue BuildNestedIssues(ui32 depth) { + NYql::TIssue issue("Testing nested issue"); + for (ui32 i = 0; i + 1 < depth; ++i) { + NYql::TIssue nestedIssue(TStringBuilder() << "Nested issue " << i); + nestedIssue.AddSubIssue(MakeIntrusive(issue)); + issue = nestedIssue; + } + return issue; + } + + Y_UNIT_TEST(TestCheckNestingDepth) { + constexpr ui32 depth = 30; + NYql::NIssue::NProto::IssueMessage nestedProto; + NYql::IssueToMessage(BuildNestedIssues(depth), &nestedProto); + + UNIT_ASSERT_C(!CheckNestingDepth(nestedProto, depth - 1), depth - 1); + UNIT_ASSERT_C(CheckNestingDepth(nestedProto, depth), depth); + UNIT_ASSERT_C(CheckNestingDepth(nestedProto, depth + 1), depth + 1); + } + + Y_UNIT_TEST(TestTruncateIssues) { + constexpr ui32 depth = 30; + NYql::TIssue issue = BuildNestedIssues(depth); + + constexpr ui32 maxLevels = 10; + constexpr ui32 keepTailLevels = 3; + const auto truncated = TruncateIssues({issue}, maxLevels, keepTailLevels); + UNIT_ASSERT_VALUES_EQUAL(truncated.Size(), 1); + + NYql::NIssue::NProto::IssueMessage nestedProto; + NYql::IssueToMessage(*truncated.begin(), &nestedProto); + UNIT_ASSERT(CheckNestingDepth(nestedProto, maxLevels + keepTailLevels)); + } + + Y_UNIT_TEST(TestValidateResultSetColumns) { + constexpr ui32 depth = 30; + + Ydb::Type type; + type.set_type_id(Ydb::Type::INT32); + for (ui32 i = 0; i < depth; ++i) { + Ydb::Type nestedType; + *nestedType.mutable_optional_type()->mutable_item() = type; + type = nestedType; + } + + Ydb::Query::ResultSetMeta meta; + *meta.add_columns()->mutable_type() = type; + + auto issues = ValidateResultSetColumns(meta.columns(), 2 * depth); + UNIT_ASSERT_C(!issues.Empty(), 2 * depth); + + issues = ValidateResultSetColumns(meta.columns(), 2 * depth + 1); + UNIT_ASSERT_C(issues.Empty(), issues.ToOneLineString()); + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/kqp/federated_query/ut/ya.make b/ydb/core/kqp/federated_query/ut/ya.make new file mode 100644 index 000000000000..6570bfc7a3fe --- /dev/null +++ b/ydb/core/kqp/federated_query/ut/ya.make @@ -0,0 +1,20 @@ +UNITTEST_FOR(ydb/core/kqp/federated_query) + +PEERDIR( + ydb/core/kqp/federated_query + ydb/public/api/protos + yql/essentials/minikql/comp_nodes/llvm16 + yql/essentials/public/issue/protos + yql/essentials/public/udf/service/stub + yql/essentials/sql/pg_dummy + yt/yql/providers/yt/comp_nodes/dq/llvm16 + yt/yql/providers/yt/comp_nodes/llvm16 +) + +SRCS( + kqp_federated_query_helpers_ut.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/kqp/federated_query/ya.make b/ydb/core/kqp/federated_query/ya.make index 10569fe95418..d8fc5c4529b6 100644 --- a/ydb/core/kqp/federated_query/ya.make +++ b/ydb/core/kqp/federated_query/ya.make @@ -12,7 +12,9 @@ PEERDIR( ydb/library/db_pool/protos ydb/library/yql/providers/common/http_gateway ydb/library/yql/providers/generic/connector/libcpp + ydb/library/yql/providers/s3/actors_factory ydb/library/yql/providers/solomon/gateway + yql/essentials/public/issue yt/yql/providers/yt/gateway/native yt/yql/providers/yt/lib/yt_download ) @@ -20,3 +22,7 @@ PEERDIR( YQL_LAST_ABI_VERSION() END() + +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp index 0b5543a63d7d..79976a574828 100644 --- a/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp +++ b/ydb/core/kqp/run_script_actor/kqp_run_script_actor.cpp @@ -1,19 +1,20 @@ #include "kqp_run_script_actor.h" -#include #include #include #include #include +#include #include #include +#include #include -#include - #include #include #include #include +#include + #include #include @@ -384,6 +385,16 @@ class TRunScriptActor : public NActors::TActorBootstrapped { Ydb::Query::Internal::ResultSetMeta meta; if (newResultSet) { *meta.mutable_columns() = ev->Get()->Record.GetResultSet().columns(); + + if (const auto& issues = NKikimr::NKqp::ValidateResultSetColumns(meta.columns())) { + NYql::TIssue rootIssue(TStringBuilder() << "Invalid result set " << resultSetIndex << " columns, please contact internal support"); + for (const NYql::TIssue& issue : issues) { + rootIssue.AddSubIssue(MakeIntrusive(issue)); + } + Issues.AddIssue(rootIssue); + Finish(Ydb::StatusIds::INTERNAL_ERROR); + return; + } } if (resultSetInfo.Truncated) { meta.set_truncated(true); @@ -450,6 +461,7 @@ class TRunScriptActor : public NActors::TActorBootstrapped { const auto& issueMessage = record.GetResponse().GetQueryIssues(); NYql::IssuesFromMessage(issueMessage, Issues); + Issues = TruncateIssues(Issues); if (record.GetYdbStatus() == Ydb::StatusIds::TIMEOUT) { const TDuration timeout = GetQueryTimeout(NKikimrKqp::QUERY_TYPE_SQL_GENERIC_SCRIPT, Request.GetRequest().GetTimeoutMs(), {}, QueryServiceConfig); diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c25be1a6d3f3..f1ca37294cf4 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -41,10 +41,11 @@ #include #include #include -#include #include #include +#include #include +#include #include #include #include diff --git a/ydb/services/fq/ut_integration/fq_ut.cpp b/ydb/services/fq/ut_integration/fq_ut.cpp index c8210a20899b..ac2a0682c07c 100644 --- a/ydb/services/fq/ut_integration/fq_ut.cpp +++ b/ydb/services/fq/ut_integration/fq_ut.cpp @@ -8,7 +8,7 @@ #include #include -#include +#include #include diff --git a/ydb/tests/fq/s3/test_empty.py b/ydb/tests/fq/s3/test_empty.py deleted file mode 100644 index 9e582b61bdbb..000000000000 --- a/ydb/tests/fq/s3/test_empty.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import pytest - -import ydb.public.api.protos.draft.fq_pb2 as fq - -from ydb.tests.tools.fq_runner.fq_client import FederatedQueryException -from ydb.tests.tools.fq_runner.kikimr_utils import yq_all - - -class TestS3(object): - @yq_all - @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) - def test_empty(self, client): - try: - client.create_query("simple", "", type=fq.QueryContent.QueryType.ANALYTICS).result.query_id - except FederatedQueryException as e: - assert "message: \"text\\\'s length is not in [1; 102400]" in e.args[0] - return - assert False diff --git a/ydb/tests/fq/s3/test_validation.py b/ydb/tests/fq/s3/test_validation.py new file mode 100644 index 000000000000..66921f2a17da --- /dev/null +++ b/ydb/tests/fq/s3/test_validation.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import pytest + +import ydb.public.api.protos.draft.fq_pb2 as fq + +from ydb.tests.tools.fq_runner.fq_client import FederatedQueryException +from ydb.tests.tools.fq_runner.kikimr_utils import yq_all + + +class TestS3(object): + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_empty(self, client): + try: + client.create_query("simple", "", type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + except FederatedQueryException as e: + assert "message: \"text\\\'s length is not in [1; 102400]" in e.args[0] + return + assert False + + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_nested_issues(self, client): + sql = "$t_0 = SELECT 1 - \"f\" AS data;\n" + + number_rows = 100 + for i in range(number_rows): + sql += f"$t_{i + 1} = SELECT data FROM $t_{i} WHERE data != {i};\n" + + sql += f"SELECT data FROM $t_{number_rows};" + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.FAILED) + + describe_result = client.describe_query(query_id).result + describe_string = "{}".format(describe_result) + assert "(skipped levels)" in describe_string, describe_string + assert "Cannot substract" in describe_string, describe_string + + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_nested_type(self, client): + sql = "$t_0 = SELECT 1 AS data;\n" + + number_rows = 300 + for i in range(number_rows): + sql += f"$t_{i + 1} = SELECT <|data:data|> AS data FROM $t_{i};\n" + + sql += f"SELECT data FROM $t_{number_rows};" + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.FAILED) + + describe_result = client.describe_query(query_id).result + describe_string = "{}".format(describe_result) + assert "Invalid result set" in describe_string, describe_string + assert "Nesting depth of type for result column" in describe_string, describe_string diff --git a/ydb/tests/fq/s3/ya.make b/ydb/tests/fq/s3/ya.make index d1bf30802f05..14ddb301a835 100644 --- a/ydb/tests/fq/s3/ya.make +++ b/ydb/tests/fq/s3/ya.make @@ -26,7 +26,6 @@ TEST_SRCS( test_bindings_1.py test_compressions.py test_early_finish.py - test_empty.py test_explicit_partitioning_0.py test_explicit_partitioning_1.py test_format_setting.py @@ -41,6 +40,7 @@ TEST_SRCS( test_size_limit.py test_statistics.py test_test_connection.py + test_validation.py test_ydb_over_fq.py test_yq_v2.py ) From 8a403ae1b86da17f78c601a054b8e87764cf7bda Mon Sep 17 00:00:00 2001 From: Oleg Doronin Date: Tue, 8 Jul 2025 16:52:21 +0000 Subject: [PATCH 62/62] include has been fixed --- ydb/library/yql/tools/dqrun/dqrun.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/library/yql/tools/dqrun/dqrun.cpp b/ydb/library/yql/tools/dqrun/dqrun.cpp index 34f1b2485e56..adadde751b01 100644 --- a/ydb/library/yql/tools/dqrun/dqrun.cpp +++ b/ydb/library/yql/tools/dqrun/dqrun.cpp @@ -84,7 +84,7 @@ #include #include -#include +#include #include #include #include
TierTimeError