From 758d7e16618ee9a1747477196b1369ac1a568750 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Mon, 7 Jul 2025 16:23:59 +0300 Subject: [PATCH 1/6] change memory control in cs data fetcher --- .../tx/columnshard/data_reader/contexts.cpp | 4 +- .../tx/columnshard/data_reader/contexts.h | 55 +++++++++---------- ydb/core/tx/columnshard/data_reader/fetcher.h | 2 +- .../reader/common_reader/iterator/context.cpp | 3 +- .../reader/plain_reader/iterator/interval.cpp | 3 +- .../reader/simple_reader/iterator/source.cpp | 3 +- .../limiter/grouped_memory/usage/abstract.h | 42 +++++++++++--- .../tx/limiter/grouped_memory/usage/service.h | 13 ----- 8 files changed, 68 insertions(+), 57 deletions(-) diff --git a/ydb/core/tx/columnshard/data_reader/contexts.cpp b/ydb/core/tx/columnshard/data_reader/contexts.cpp index 8b3fb0c1ec64..1aadbf77d6b0 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.cpp +++ b/ydb/core/tx/columnshard/data_reader/contexts.cpp @@ -15,10 +15,10 @@ IFetchingStep::EStepResult IFetchingStep::Execute(const std::shared_ptr& portions, const std::shared_ptr& versions, const NBlobOperations::EConsumer consumer, const TString& externalTaskId, - const std::optional& memoryProcessInfo) + const std::shared_ptr& memoryGroupGuard) : Consumer(consumer) , ExternalTaskId(externalTaskId) - , MemoryProcessInfo(memoryProcessInfo ? *memoryProcessInfo : TFetcherMemoryProcessInfo()) + , MemoryGroupGuard(memoryGroupGuard) { AFL_VERIFY(portions.size()); ActualSchema = versions->GetLastSchema(); diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h index a01bde60e0d7..167e7c9f00d9 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.h +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -21,41 +21,26 @@ enum class EFetchingStage : ui32 { Error }; -class TFetcherMemoryProcessInfo { -private: - static inline TAtomicCounter Counter = 0; - ui64 MemoryProcessId = Counter.Inc(); - -public: - ui64 GetMemoryProcessId() const { - return MemoryProcessId; - } -}; - class TCurrentContext: TMoveOnly { private: std::optional> Accessors; YDB_READONLY_DEF(std::vector>, ResourceGuards); - std::shared_ptr MemoryProcessGuard; - std::shared_ptr MemoryProcessScopeGuard; - std::shared_ptr MemoryProcessGroupGuard; - TFetcherMemoryProcessInfo MemoryProcessInfo; + std::shared_ptr MemoryGroupGuard; std::optional Blobs; std::optional> AssembledData; + inline static TAtomicCounter MemoryProcessIdCounter = 0; public: ui64 GetMemoryProcessId() const { - return MemoryProcessInfo.GetMemoryProcessId(); + return MemoryGroupGuard->GetProcessGuard()->GetProcessId(); } ui64 GetMemoryScopeId() const { - AFL_VERIFY(!!MemoryProcessScopeGuard); - return MemoryProcessScopeGuard->GetScopeId(); + return MemoryGroupGuard->GetScopeGuard()->GetScopeId(); } ui64 GetMemoryGroupId() const { - AFL_VERIFY(!!MemoryProcessGroupGuard); - return MemoryProcessGroupGuard->GetGroupId(); + return MemoryGroupGuard->GetGroupGuard()->GetGroupId(); } void SetBlobs(NBlobOperations::NRead::TCompositeReadBlobs&& blobs) { @@ -91,15 +76,20 @@ class TCurrentContext: TMoveOnly { return result; } - TCurrentContext(const TFetcherMemoryProcessInfo& memoryProcessInfo) - : MemoryProcessInfo(memoryProcessInfo) - { + static std::shared_ptr BuildGroupGuard() { static std::shared_ptr stageFeatures = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - MemoryProcessGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(GetMemoryProcessId(), { stageFeatures }); - MemoryProcessScopeGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildScopeGuard(GetMemoryProcessId(), 1); - MemoryProcessGroupGuard = NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildGroupGuard(GetMemoryProcessId(), 1); + auto processGuard = + NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(MemoryProcessIdCounter.Inc(), { stageFeatures }); + auto scopeGuard = processGuard->BuildScopeGuard(1); + auto groupGuard = scopeGuard->BuildGroupGuard(); + return std::make_shared(groupGuard, scopeGuard, processGuard); + } + + TCurrentContext(std::shared_ptr&& memoryGroupGuard) + : MemoryGroupGuard(memoryGroupGuard ? std::move(memoryGroupGuard) : BuildGroupGuard()) + { } void SetPortionAccessors(std::vector&& acc) { @@ -261,12 +251,21 @@ class TRequestInput { YDB_READONLY_DEF(std::shared_ptr, ActualSchema); YDB_READONLY(NBlobOperations::EConsumer, Consumer, NBlobOperations::EConsumer::UNDEFINED); YDB_READONLY_DEF(TString, ExternalTaskId); - YDB_READONLY_DEF(TFetcherMemoryProcessInfo, MemoryProcessInfo); + std::shared_ptr MemoryGroupGuard; public: TRequestInput(const std::vector& portions, const std::shared_ptr& versions, const NBlobOperations::EConsumer consumer, const TString& externalTaskId, - const std::optional& memoryProcessInfo = std::nullopt); + const std::shared_ptr& memoryGroupGuard = nullptr); + + std::shared_ptr ExtractGroupGuardOptional() { + if (!MemoryGroupGuard) { + return nullptr; + } + auto result = std::move(MemoryGroupGuard); + MemoryGroupGuard.reset(); + return result; + } }; } // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index 50bb0f4ee11a..eabfefe216f2 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -80,7 +80,7 @@ class TPortionsDataFetcher: TNonCopyable { , ClassCounters(Singleton()->GetClassCounters(Callback->GetClassName())) , Guard(ClassCounters->GetGuard(EFetchingStage::Created)) , Script(script) - , CurrentContext(input.GetMemoryProcessInfo()) + , CurrentContext(input.ExtractGroupGuardOptional()) , Environment(environment) , ConveyorCategory(conveyorCategory) { diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/context.cpp index 24fadcc32d61..4b805aef4f6a 100644 --- a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/context.cpp @@ -41,8 +41,7 @@ TSpecialReadContext::TSpecialReadContext(const std::shared_ptr& co NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildStageFeatures(stagePrefix + "::MERGE", kffMerge * TGlobalLimits::ScanMemoryLimit) }; ProcessMemoryGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(ReadMetadata->GetTxId(), stages); - ProcessScopeGuard = - NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildScopeGuard(ReadMetadata->GetTxId(), GetCommonContext()->GetScanId()); + ProcessScopeGuard = ProcessMemoryGuard->BuildScopeGuard(GetCommonContext()->GetScanId()); auto readSchema = ReadMetadata->GetResultSchema(); SpecColumns = std::make_shared(TIndexInfo::GetSnapshotColumnIdsSet(), readSchema); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp index a431b81077f4..00b5e6f89dfa 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/interval.cpp @@ -38,8 +38,7 @@ TFetchingInterval::TFetchingInterval(const NArrow::NMerger::TSortableBatchPositi , TaskGuard(Context->GetCommonContext()->GetCounters().GetResourcesAllocationTasksGuard()) , Sources(sources) , IntervalIdx(intervalIdx) - , IntervalGroupGuard(NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard( - Context->GetProcessMemoryControlId(), context->GetCommonContext()->GetScanId())) + , IntervalGroupGuard(Context->GetProcessScopeGuard()->BuildGroupGuard()) , IntervalStateGuard(Context->GetCommonContext()->GetCounters().CreateIntervalStateGuard()) { AFL_VERIFY(Sources.size()); for (auto&& [_, i] : Sources) { diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp index c8c331f3e882..e37e0b92a82d 100644 --- a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp +++ b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp @@ -33,8 +33,7 @@ void IDataSource::StartProcessing(const std::shared_ptr& sourcePtr) InitStageData(std::make_unique( GetContext()->GetReadMetadata()->GetProgram().GetChainVerified()->HasAggregations(), sourcePtr->GetRecordsCount())); ProcessingStarted = true; - SourceGroupGuard = NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildGroupGuard( - GetContext()->GetProcessMemoryControlId(), GetContext()->GetCommonContext()->GetScanId()); + SourceGroupGuard = GetContext()->GetProcessScopeGuard()->BuildGroupGuard(); SetMemoryGroupId(SourceGroupGuard->GetGroupId()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_SCAN)("InitFetchingPlan", FetchingPlan->DebugString())("source_idx", GetSourceIdx()); // NActors::TLogContextGuard logGuard(NActors::TLogContextBuilder::Build()("source", SourceIdx)("method", "InitFetchingPlan")); diff --git a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h index eeb4169eec7d..180759f33d5e 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h @@ -25,27 +25,55 @@ class TGroupGuard { ~TGroupGuard(); }; -class TProcessGuard { +class TScopeGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, ScopeId, 0); public: - TProcessGuard(const NActors::TActorId& actorId, const ui64 processId, const std::vector>& stages); + TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId); - ~TProcessGuard(); + std::shared_ptr BuildGroupGuard() const { + static TAtomicCounter counter = 0; + return std::make_shared(ActorId, ProcessId, ScopeId, counter.Inc()); + } + + ~TScopeGuard(); }; -class TScopeGuard { +class TProcessGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0); - YDB_READONLY(ui64, ScopeId, 0); public: - TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId); + TProcessGuard(const NActors::TActorId& actorId, const ui64 processId, const std::vector>& stages); - ~TScopeGuard(); + std::shared_ptr BuildScopeGuard(const ui32 scopeId) const { + return std::make_shared(ActorId, ProcessId, scopeId); + } + + ~TProcessGuard(); +}; + +class TFullGroupGuard : NNonCopyable::TMoveOnly { +private: + YDB_READONLY_DEF(std::shared_ptr, GroupGuard); + YDB_READONLY_DEF(std::shared_ptr, ScopeGuard); + YDB_READONLY_DEF(std::shared_ptr, ProcessGuard); + +public: + TFullGroupGuard( + std::shared_ptr groupGuard, std::shared_ptr scopeGuard, std::shared_ptr processGuard) + : GroupGuard(std::move(groupGuard)) + , ScopeGuard(std::move(scopeGuard)) + , ProcessGuard(std::move(processGuard)) + { + AFL_VERIFY(GroupGuard); + AFL_VERIFY(ScopeGuard); + AFL_VERIFY(ProcessGuard); + } }; class TAllocationGuard { diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index 2a18744bf612..e12253242988 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -45,19 +45,6 @@ class TServiceOperatorImpl { return Singleton()->DefaultStageFeatures; } - static std::shared_ptr BuildGroupGuard(const ui64 processId, const ui32 scopeId) { - static TAtomicCounter counter = 0; - auto& context = NActors::TActorContext::AsActorContext(); - const NActors::TActorId& selfId = context.SelfID; - return std::make_shared(MakeServiceId(selfId.NodeId()), processId, scopeId, counter.Inc()); - } - - static std::shared_ptr BuildScopeGuard(const ui64 processId, const ui32 scopeId) { - auto& context = NActors::TActorContext::AsActorContext(); - const NActors::TActorId& selfId = context.SelfID; - return std::make_shared(MakeServiceId(selfId.NodeId()), processId, scopeId); - } - static std::shared_ptr BuildProcessGuard(const ui64 processId, const std::vector>& stages) { auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; From d5817bdef7b8b55211b060cd814595da371539f8 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 8 Jul 2025 12:32:46 +0300 Subject: [PATCH 2/6] fix memory management --- ydb/core/tx/columnshard/columnshard_impl.cpp | 17 +++++-- .../tx/columnshard/data_reader/contexts.cpp | 4 +- .../tx/columnshard/data_reader/contexts.h | 44 ++++++++----------- ydb/core/tx/columnshard/data_reader/fetcher.h | 2 +- .../engines/reader/sys_view/chunks/chunks.cpp | 10 +++-- .../limiter/grouped_memory/usage/abstract.h | 19 -------- .../tx/limiter/grouped_memory/usage/service.h | 17 ++++++- 7 files changed, 56 insertions(+), 57 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index ea00a286ef0a..6e532dda74f7 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -628,8 +628,11 @@ void TColumnShard::StartCompaction(const std::shared_ptrGetVersionedIndexReadonlyCopy(); + static std::shared_ptr stageFeatures = + NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput(compaction.GetSwitchedPortions(), actualIndexInfo, - NOlap::NBlobOperations::EConsumer::GENERAL_COMPACTION, compaction.GetTaskIdentifier()); + NOlap::NBlobOperations::EConsumer::GENERAL_COMPACTION, compaction.GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); NOlap::NDataFetcher::TPortionsDataFetcher::StartFullPortionsFetching(std::move(rInput), std::make_shared( @@ -749,8 +752,11 @@ bool TColumnShard::SetupTtl() { auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); for (auto&& i : indexChanges) { i->Start(*this); + static std::shared_ptr stageFeatures = + NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput( - i->GetPortionsInfo(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::TTL, i->GetTaskIdentifier()); + i->GetPortionsInfo(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::TTL, i->GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); if (i->NeedConstruction()) { NOlap::NDataFetcher::TPortionsDataFetcher::StartFullPortionsFetching(std::move(rInput), @@ -790,8 +796,11 @@ void TColumnShard::SetupCleanupPortions() { changes->Start(*this); auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); - NOlap::NDataFetcher::TRequestInput rInput( - changes->GetPortionsToAccess(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier()); + static std::shared_ptr stageFeatures = + NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); + NOlap::NDataFetcher::TRequestInput rInput(changes->GetPortionsToAccess(), actualIndexInfo, + NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching(std::move(rInput), std::make_shared( diff --git a/ydb/core/tx/columnshard/data_reader/contexts.cpp b/ydb/core/tx/columnshard/data_reader/contexts.cpp index 1aadbf77d6b0..b47f3ebb4963 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.cpp +++ b/ydb/core/tx/columnshard/data_reader/contexts.cpp @@ -15,10 +15,10 @@ IFetchingStep::EStepResult IFetchingStep::Execute(const std::shared_ptr& portions, const std::shared_ptr& versions, const NBlobOperations::EConsumer consumer, const TString& externalTaskId, - const std::shared_ptr& memoryGroupGuard) + const std::shared_ptr& memoryProcessGuard) : Consumer(consumer) , ExternalTaskId(externalTaskId) - , MemoryGroupGuard(memoryGroupGuard) + , MemoryProcessGuard(memoryProcessGuard) { AFL_VERIFY(portions.size()); ActualSchema = versions->GetLastSchema(); diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h index 167e7c9f00d9..e13d2575b8d6 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.h +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -25,22 +25,24 @@ class TCurrentContext: TMoveOnly { private: std::optional> Accessors; YDB_READONLY_DEF(std::vector>, ResourceGuards); - std::shared_ptr MemoryGroupGuard; + std::shared_ptr MemoryProcessGuard; + std::shared_ptr MemoryScopeGuard; + std::shared_ptr MemoryGroupGuard; std::optional Blobs; std::optional> AssembledData; inline static TAtomicCounter MemoryProcessIdCounter = 0; public: ui64 GetMemoryProcessId() const { - return MemoryGroupGuard->GetProcessGuard()->GetProcessId(); + return MemoryProcessGuard->GetProcessId(); } ui64 GetMemoryScopeId() const { - return MemoryGroupGuard->GetScopeGuard()->GetScopeId(); + return MemoryScopeGuard->GetScopeId(); } ui64 GetMemoryGroupId() const { - return MemoryGroupGuard->GetGroupGuard()->GetGroupId(); + return MemoryGroupGuard->GetGroupId(); } void SetBlobs(NBlobOperations::NRead::TCompositeReadBlobs&& blobs) { @@ -76,20 +78,12 @@ class TCurrentContext: TMoveOnly { return result; } - static std::shared_ptr BuildGroupGuard() { - static std::shared_ptr stageFeatures = - NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - - auto processGuard = - NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(MemoryProcessIdCounter.Inc(), { stageFeatures }); - auto scopeGuard = processGuard->BuildScopeGuard(1); - auto groupGuard = scopeGuard->BuildGroupGuard(); - return std::make_shared(groupGuard, scopeGuard, processGuard); - } - - TCurrentContext(std::shared_ptr&& memoryGroupGuard) - : MemoryGroupGuard(memoryGroupGuard ? std::move(memoryGroupGuard) : BuildGroupGuard()) + TCurrentContext(std::shared_ptr&& memoryProcessGuard) + : MemoryProcessGuard(std::move(memoryProcessGuard)) { + AFL_VERIFY(MemoryProcessGuard); + MemoryScopeGuard = MemoryProcessGuard->BuildScopeGuard(1); + MemoryGroupGuard = MemoryScopeGuard->BuildGroupGuard(); } void SetPortionAccessors(std::vector&& acc) { @@ -251,19 +245,17 @@ class TRequestInput { YDB_READONLY_DEF(std::shared_ptr, ActualSchema); YDB_READONLY(NBlobOperations::EConsumer, Consumer, NBlobOperations::EConsumer::UNDEFINED); YDB_READONLY_DEF(TString, ExternalTaskId); - std::shared_ptr MemoryGroupGuard; + std::shared_ptr MemoryProcessGuard; public: TRequestInput(const std::vector& portions, const std::shared_ptr& versions, const NBlobOperations::EConsumer consumer, const TString& externalTaskId, - const std::shared_ptr& memoryGroupGuard = nullptr); - - std::shared_ptr ExtractGroupGuardOptional() { - if (!MemoryGroupGuard) { - return nullptr; - } - auto result = std::move(MemoryGroupGuard); - MemoryGroupGuard.reset(); + const std::shared_ptr& memoryProcessGuard); + + std::shared_ptr ExtractProcessGuardVerified() { + AFL_VERIFY(MemoryProcessGuard); + auto result = std::move(MemoryProcessGuard); + MemoryProcessGuard.reset(); return result; } }; diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index eabfefe216f2..39fb8ade141c 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -80,7 +80,7 @@ class TPortionsDataFetcher: TNonCopyable { , ClassCounters(Singleton()->GetClassCounters(Callback->GetClassName())) , Guard(ClassCounters->GetGuard(EFetchingStage::Created)) , Script(script) - , CurrentContext(input.ExtractGroupGuardOptional()) + , CurrentContext(input.ExtractProcessGuardVerified()) , Environment(environment) , ConveyorCategory(conveyorCategory) { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index d602958b80ee..c901de4dfce6 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -226,6 +226,8 @@ class TFetchingExecutor: public NOlap::NDataFetcher::IFetchCallback { } // namespace TConclusionStatus TStatsIterator::Start() { + static std::shared_ptr stageFeatures = + NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); std::vector portions; std::shared_ptr actualIndexInfo; auto env = std::make_shared(Context->GetDataAccessorsManager(), Context->GetStoragesManager()); @@ -236,8 +238,9 @@ TConclusionStatus TStatsIterator::Start() { if (!actualIndexInfo) { actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); } - NOlap::NDataFetcher::TRequestInput rInput( - std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId())); + auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); + NOlap::NDataFetcher::TRequestInput rInput(std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, + ::ToString(ReadMetadata->GetTxId()), processGuard); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( std::move(rInput), std::make_shared(Context), env, NConveyorComposite::ESpecialTaskCategory::Scan); portions.clear(); @@ -248,8 +251,9 @@ TConclusionStatus TStatsIterator::Start() { if (!actualIndexInfo) { actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); } + auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput( - std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId())); + std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId()), processGuard); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( std::move(rInput), std::make_shared(Context), env, NConveyorComposite::ESpecialTaskCategory::Scan); } diff --git a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h index 180759f33d5e..cc6c0bc7ab51 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h @@ -57,25 +57,6 @@ class TProcessGuard { ~TProcessGuard(); }; -class TFullGroupGuard : NNonCopyable::TMoveOnly { -private: - YDB_READONLY_DEF(std::shared_ptr, GroupGuard); - YDB_READONLY_DEF(std::shared_ptr, ScopeGuard); - YDB_READONLY_DEF(std::shared_ptr, ProcessGuard); - -public: - TFullGroupGuard( - std::shared_ptr groupGuard, std::shared_ptr scopeGuard, std::shared_ptr processGuard) - : GroupGuard(std::move(groupGuard)) - , ScopeGuard(std::move(scopeGuard)) - , ProcessGuard(std::move(processGuard)) - { - AFL_VERIFY(GroupGuard); - AFL_VERIFY(ScopeGuard); - AFL_VERIFY(ProcessGuard); - } -}; - class TAllocationGuard { private: const NActors::TActorId ActorId; diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index e12253242988..07bbbc2dad4e 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -13,6 +13,7 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { template class TServiceOperatorImpl { private: + TAtomicCounter NextProcessId = 0; TConfig ServiceConfig = TConfig::BuildDisabledConfig(); std::shared_ptr Counters; std::shared_ptr DefaultStageFeatures = @@ -45,10 +46,20 @@ class TServiceOperatorImpl { return Singleton()->DefaultStageFeatures; } - static std::shared_ptr BuildProcessGuard(const ui64 processId, const std::vector>& stages) { + static std::shared_ptr BuildProcessGuard( + const std::optional processId, const std::vector>& stages) { + ui64 localProcessId; + if constexpr (TMemoryLimiterPolicy::ExternalProcessIdAllocation) { + AFL_VERIFY(processId); + localProcessId = *processId; + } else { + AFL_VERIFY(!processId); + localProcessId = Singleton()->NextProcessId.Val(); + Singleton()->NextProcessId.Inc(); + } auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; - return std::make_shared(MakeServiceId(selfId.NodeId()), processId, stages); + return std::make_shared(MakeServiceId(selfId.NodeId()), localProcessId, stages); } static bool SendToAllocation(const ui64 processId, const ui64 scopeId, const ui64 groupId, @@ -83,6 +94,7 @@ class TServiceOperatorImpl { class TScanMemoryLimiterPolicy { public: static const inline TString Name = "Scan"; + static const bool ExternalProcessIdAllocation = true; }; using TScanMemoryLimiterOperator = TServiceOperatorImpl; @@ -90,6 +102,7 @@ using TScanMemoryLimiterOperator = TServiceOperatorImpl; From 141226b04c03dedb59eda7db26bbe2892bdc918f Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 8 Jul 2025 12:52:47 +0300 Subject: [PATCH 3/6] fix --- ydb/core/tx/columnshard/data_reader/fetcher.h | 4 ++-- ydb/core/tx/limiter/grouped_memory/usage/service.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index 39fb8ade141c..1c7c814d4418 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -52,7 +52,7 @@ class TCounters: public NColumnShard::TCommonCountersOwner { class TPortionsDataFetcher: TNonCopyable { private: - const TRequestInput Input; + TRequestInput Input; const std::shared_ptr Callback; std::shared_ptr ClassCounters; NCounters::TStateSignalsOperator::TGuard Guard; @@ -80,7 +80,7 @@ class TPortionsDataFetcher: TNonCopyable { , ClassCounters(Singleton()->GetClassCounters(Callback->GetClassName())) , Guard(ClassCounters->GetGuard(EFetchingStage::Created)) , Script(script) - , CurrentContext(input.ExtractProcessGuardVerified()) + , CurrentContext(Input.ExtractProcessGuardVerified()) , Environment(environment) , ConveyorCategory(conveyorCategory) { diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index 07bbbc2dad4e..8345b1eb5f3b 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -102,7 +102,7 @@ using TScanMemoryLimiterOperator = TServiceOperatorImpl; From efd008e0431b2f473b8da48d4902a319723cd279 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 8 Jul 2025 13:51:29 +0300 Subject: [PATCH 4/6] post-review --- ydb/core/tx/columnshard/columnshard_impl.cpp | 6 ++-- .../tx/columnshard/data_reader/contexts.h | 17 ++++------- ydb/core/tx/columnshard/data_reader/fetcher.h | 2 +- .../engines/reader/sys_view/chunks/chunks.cpp | 3 +- .../limiter/grouped_memory/usage/abstract.h | 28 +++++++++++-------- .../tx/limiter/grouped_memory/usage/service.h | 27 +++++++++--------- 6 files changed, 40 insertions(+), 43 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 6e532dda74f7..0214b1d2e2f8 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -630,7 +630,7 @@ void TColumnShard::StartCompaction(const std::shared_ptrGetVersionedIndexReadonlyCopy(); static std::shared_ptr stageFeatures = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard({ stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput(compaction.GetSwitchedPortions(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::GENERAL_COMPACTION, compaction.GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); @@ -754,7 +754,7 @@ bool TColumnShard::SetupTtl() { i->Start(*this); static std::shared_ptr stageFeatures = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard({ stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput( i->GetPortionsInfo(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::TTL, i->GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); @@ -798,7 +798,7 @@ void TColumnShard::SetupCleanupPortions() { auto actualIndexInfo = TablesManager.GetPrimaryIndex()->GetVersionedIndexReadonlyCopy(); static std::shared_ptr stageFeatures = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildStageFeatures("DEFAULT", 1000000000); - auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); + auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard({ stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput(changes->GetPortionsToAccess(), actualIndexInfo, NOlap::NBlobOperations::EConsumer::CLEANUP_PORTIONS, changes->GetTaskIdentifier(), processGuard); auto env = std::make_shared(DataAccessorsManager.GetObjectPtrVerified(), StoragesManager); diff --git a/ydb/core/tx/columnshard/data_reader/contexts.h b/ydb/core/tx/columnshard/data_reader/contexts.h index e13d2575b8d6..83a08a23ee40 100644 --- a/ydb/core/tx/columnshard/data_reader/contexts.h +++ b/ydb/core/tx/columnshard/data_reader/contexts.h @@ -30,7 +30,7 @@ class TCurrentContext: TMoveOnly { std::shared_ptr MemoryGroupGuard; std::optional Blobs; std::optional> AssembledData; - inline static TAtomicCounter MemoryProcessIdCounter = 0; + inline static TAtomicCounter MemoryScopeIdCounter = 0; public: ui64 GetMemoryProcessId() const { @@ -78,11 +78,11 @@ class TCurrentContext: TMoveOnly { return result; } - TCurrentContext(std::shared_ptr&& memoryProcessGuard) - : MemoryProcessGuard(std::move(memoryProcessGuard)) + TCurrentContext(const std::shared_ptr& memoryProcessGuard) + : MemoryProcessGuard(memoryProcessGuard) { AFL_VERIFY(MemoryProcessGuard); - MemoryScopeGuard = MemoryProcessGuard->BuildScopeGuard(1); + MemoryScopeGuard = MemoryProcessGuard->BuildScopeGuard(MemoryScopeIdCounter.Inc()); MemoryGroupGuard = MemoryScopeGuard->BuildGroupGuard(); } @@ -245,19 +245,12 @@ class TRequestInput { YDB_READONLY_DEF(std::shared_ptr, ActualSchema); YDB_READONLY(NBlobOperations::EConsumer, Consumer, NBlobOperations::EConsumer::UNDEFINED); YDB_READONLY_DEF(TString, ExternalTaskId); - std::shared_ptr MemoryProcessGuard; + YDB_READONLY_DEF(std::shared_ptr, MemoryProcessGuard); public: TRequestInput(const std::vector& portions, const std::shared_ptr& versions, const NBlobOperations::EConsumer consumer, const TString& externalTaskId, const std::shared_ptr& memoryProcessGuard); - - std::shared_ptr ExtractProcessGuardVerified() { - AFL_VERIFY(MemoryProcessGuard); - auto result = std::move(MemoryProcessGuard); - MemoryProcessGuard.reset(); - return result; - } }; } // namespace NKikimr::NOlap::NDataFetcher diff --git a/ydb/core/tx/columnshard/data_reader/fetcher.h b/ydb/core/tx/columnshard/data_reader/fetcher.h index 1c7c814d4418..53c6e55a3569 100644 --- a/ydb/core/tx/columnshard/data_reader/fetcher.h +++ b/ydb/core/tx/columnshard/data_reader/fetcher.h @@ -80,7 +80,7 @@ class TPortionsDataFetcher: TNonCopyable { , ClassCounters(Singleton()->GetClassCounters(Callback->GetClassName())) , Guard(ClassCounters->GetGuard(EFetchingStage::Created)) , Script(script) - , CurrentContext(Input.ExtractProcessGuardVerified()) + , CurrentContext(Input.GetMemoryProcessGuard()) , Environment(environment) , ConveyorCategory(conveyorCategory) { diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index c901de4dfce6..5f532553943a 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -231,6 +231,7 @@ TConclusionStatus TStatsIterator::Start() { std::vector portions; std::shared_ptr actualIndexInfo; auto env = std::make_shared(Context->GetDataAccessorsManager(), Context->GetStoragesManager()); + auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(Context->GetScanId(), { stageFeatures }); for (auto&& i : IndexGranules) { for (auto&& p : i.GetPortions()) { portions.emplace_back(p); @@ -238,7 +239,6 @@ TConclusionStatus TStatsIterator::Start() { if (!actualIndexInfo) { actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); } - auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput(std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId()), processGuard); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( @@ -251,7 +251,6 @@ TConclusionStatus TStatsIterator::Start() { if (!actualIndexInfo) { actualIndexInfo = ReadMetadata->GetIndexVersionsPtr(); } - auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(std::nullopt, { stageFeatures }); NOlap::NDataFetcher::TRequestInput rInput( std::move(portions), actualIndexInfo, NOlap::NBlobOperations::EConsumer::SYS_VIEW_SCAN, ::ToString(ReadMetadata->GetTxId()), processGuard); NOlap::NDataFetcher::TPortionsDataFetcher::StartAccessorPortionsFetching( diff --git a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h index cc6c0bc7ab51..6dbd78ce9d0a 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/abstract.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/abstract.h @@ -12,6 +12,10 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { +class TGroupGuard; +class TScopeGuard; +class TProcessGuard; + class TGroupGuard { private: const NActors::TActorId ActorId; @@ -25,36 +29,36 @@ class TGroupGuard { ~TGroupGuard(); }; -class TScopeGuard { +class TProcessGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0); - YDB_READONLY(ui64, ScopeId, 0); public: - TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId); + TProcessGuard(const NActors::TActorId& actorId, const ui64 processId, const std::vector>& stages); - std::shared_ptr BuildGroupGuard() const { - static TAtomicCounter counter = 0; - return std::make_shared(ActorId, ProcessId, ScopeId, counter.Inc()); + std::shared_ptr BuildScopeGuard(const ui32 scopeId) const { + return std::make_shared(ActorId, ProcessId, scopeId); } - ~TScopeGuard(); + ~TProcessGuard(); }; -class TProcessGuard { +class TScopeGuard { private: const NActors::TActorId ActorId; YDB_READONLY(ui64, ProcessId, 0); + YDB_READONLY(ui64, ScopeId, 0); public: - TProcessGuard(const NActors::TActorId& actorId, const ui64 processId, const std::vector>& stages); + TScopeGuard(const NActors::TActorId& actorId, const ui64 processId, const ui64 scopeId); - std::shared_ptr BuildScopeGuard(const ui32 scopeId) const { - return std::make_shared(ActorId, ProcessId, scopeId); + std::shared_ptr BuildGroupGuard() const { + static TAtomicCounter counter = 0; + return std::make_shared(ActorId, ProcessId, ScopeId, counter.Inc()); } - ~TProcessGuard(); + ~TScopeGuard(); }; class TAllocationGuard { diff --git a/ydb/core/tx/limiter/grouped_memory/usage/service.h b/ydb/core/tx/limiter/grouped_memory/usage/service.h index 8345b1eb5f3b..4bae125f0d5b 100644 --- a/ydb/core/tx/limiter/grouped_memory/usage/service.h +++ b/ydb/core/tx/limiter/grouped_memory/usage/service.h @@ -13,7 +13,7 @@ namespace NKikimr::NOlap::NGroupedMemoryManager { template class TServiceOperatorImpl { private: - TAtomicCounter NextProcessId = 0; + TAtomicCounter LastProcessId = 0; TConfig ServiceConfig = TConfig::BuildDisabledConfig(); std::shared_ptr Counters; std::shared_ptr DefaultStageFeatures = @@ -46,20 +46,21 @@ class TServiceOperatorImpl { return Singleton()->DefaultStageFeatures; } - static std::shared_ptr BuildProcessGuard( - const std::optional processId, const std::vector>& stages) { - ui64 localProcessId; - if constexpr (TMemoryLimiterPolicy::ExternalProcessIdAllocation) { - AFL_VERIFY(processId); - localProcessId = *processId; - } else { - AFL_VERIFY(!processId); - localProcessId = Singleton()->NextProcessId.Val(); - Singleton()->NextProcessId.Inc(); - } + static std::shared_ptr BuildProcessGuard(const std::vector>& stages) + requires(!TMemoryLimiterPolicy::ExternalProcessIdAllocation) + { + ui64 processId = Singleton()->LastProcessId.Inc(); + auto& context = NActors::TActorContext::AsActorContext(); + const NActors::TActorId& selfId = context.SelfID; + return std::make_shared(MakeServiceId(selfId.NodeId()), processId, stages); + } + + static std::shared_ptr BuildProcessGuard(const ui64 processId, const std::vector>& stages) + requires(TMemoryLimiterPolicy::ExternalProcessIdAllocation) + { auto& context = NActors::TActorContext::AsActorContext(); const NActors::TActorId& selfId = context.SelfID; - return std::make_shared(MakeServiceId(selfId.NodeId()), localProcessId, stages); + return std::make_shared(MakeServiceId(selfId.NodeId()), processId, stages); } static bool SendToAllocation(const ui64 processId, const ui64 scopeId, const ui64 groupId, From c7976e76cabf7360f8473d9b5ede8bc9c2b9e2df Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 8 Jul 2025 17:58:54 +0300 Subject: [PATCH 5/6] fix sys-view --- .../tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp | 4 +++- ydb/core/tx/limiter/grouped_memory/service/manager.cpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 5f532553943a..18f947df2298 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -231,7 +231,9 @@ TConclusionStatus TStatsIterator::Start() { std::vector portions; std::shared_ptr actualIndexInfo; auto env = std::make_shared(Context->GetDataAccessorsManager(), Context->GetStoragesManager()); - auto processGuard = NOlap::NGroupedMemoryManager::TScanMemoryLimiterOperator::BuildProcessGuard(Context->GetScanId(), { stageFeatures }); + auto processGuard = + NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard({ stageFeatures }); + AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("aboba", "chuns")("process_id", processGuard->GetProcessId()); for (auto&& i : IndexGranules) { for (auto&& p : i.GetPortions()) { portions.emplace_back(p); diff --git a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp index a3383afc48a1..633094fe665f 100644 --- a/ydb/core/tx/limiter/grouped_memory/service/manager.cpp +++ b/ydb/core/tx/limiter/grouped_memory/service/manager.cpp @@ -96,6 +96,7 @@ void TManager::RegisterAllocation(const ui64 externalProcessId, const ui64 exter process->RegisterAllocation(externalScopeId, externalGroupId, task, stageIdx); } else { AFL_VERIFY(!task->OnAllocated(std::make_shared(externalProcessId, externalScopeId, task->GetIdentifier(), OwnerActorId, task->GetMemory()), task))( + "process", externalProcessId)("scope", externalScopeId)( "ext_group", externalGroupId)("stage_idx", stageIdx); } RefreshSignals(); From b3b26152113acb489dcd94eff8e920e15931ca61 Mon Sep 17 00:00:00 2001 From: Semyon Yentsov Date: Tue, 8 Jul 2025 20:50:24 +0300 Subject: [PATCH 6/6] clean up --- .../tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp index 18f947df2298..2ec8cd20ddcf 100644 --- a/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp +++ b/ydb/core/tx/columnshard/engines/reader/sys_view/chunks/chunks.cpp @@ -233,7 +233,6 @@ TConclusionStatus TStatsIterator::Start() { auto env = std::make_shared(Context->GetDataAccessorsManager(), Context->GetStoragesManager()); auto processGuard = NOlap::NGroupedMemoryManager::TCompMemoryLimiterOperator::BuildProcessGuard({ stageFeatures }); - AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("aboba", "chuns")("process_id", processGuard->GetProcessId()); for (auto&& i : IndexGranules) { for (auto&& p : i.GetPortions()) { portions.emplace_back(p);