Merge branch 'stable-25-1' into TR-25-1-2

Author: nshestakov

.github/config/muted_ya.txt

@@ -3,8 +3,16 @@
 ### Functionality
 
 * 17114:Improved audit logging for user management operations. The audit logs now include details about user modification actions such as password changes, user blocking, and unblocking, making it easier to troubleshoot login issues. [#17114](https://github.com/ydb-platform/ydb/pull/17114) ([flown4qqqq](https://github.com/flown4qqqq))
+* 18352:Added database audit logs in console's tablet.[#18352](https://github.com/ydb-platform/ydb/pull/18352) ([flown4qqqq](https://github.com/flown4qqqq))
+* 18298:Limited the creation of ReassignerActor to only one active instance to prevent [SelfHeal](https://ydb.tech/docs/ru/maintenance/manual/selfheal) from overloading BSC. [#18298](https://github.com/ydb-platform/ydb/pull/18298) ([Sergey Belyakov](https://github.com/serbel324))
+* 18294:Changed version format from Year.Major.Minor.Hotfix to Year.Major.Minor.Patch.Hotfix [#18294](https://github.com/ydb-platform/ydb/pull/18294) ([Sergey Belyakov](https://github.com/serbel324))
 
 ### Bug fixes
 
 * 17313:Fixed CopyTable operation to allow copying tables with all column types present in the source table, regardless of feature flag settings. This resolves an issue where copying tables with certain decimal types would fail after version downgrades. [#17313](https://github.com/ydb-platform/ydb/pull/17313) ([azevaykin](https://github.com/azevaykin))
 * 17122:Fixed an rare issue that caused client applications to hang during commit operations. The problem occurred because the `TEvDeletePartition` message could arrive before the `TEvApproveWriteQuota` message. The batch did not send TEvConsumed and this blocked the queue of write quota requests. [#17122](https://github.com/ydb-platform/ydb/pull/17122) ([Alek5andr-Kotov](https://github.com/Alek5andr-Kotov))
+* 18362:Table auto partitioning: Fixed crash when selecting split key from access samples containing a mix of full key and key prefix operations (e.g. exact/range reads). [#18362](https://github.com/ydb-platform/ydb/pull/18362) ([ijon](https://github.com/ijon))
+* 18301:Optimized memory usage in transactions with a large number of participants by changing the storage and resending mechanism for TEvReadSet messages. [#18302](https://github.com/ydb-platform/ydb/pull/18301) ([Alek5andr-Kotov](https://github.com/Alek5andr-Kotov))
+* 18296:Fixed replication continuing to consume disk space when storage was low, which caused VDisks to become read-only. [#18296](https://github.com/ydb-platform/ydb/pull/18296) ([Sergey Belyakov](https://github.com/serbel324))
+* 18271:Fix replication bug #10650 [#18271](https://github.com/ydb-platform/ydb/pull/18271) ([Alexander Rutkovsky](https://github.com/alexvru))
+* 18231:Fix segfault that could happen while retrying Whiteboard requests. [#18231](https://github.com/ydb-platform/ydb/pull/18231) ([Andrei Rykov](https://github.com/StekPerepolnen))

CHANGELOG.md

@@ -60,6 +60,10 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
 
         SendRequest(ctx);
         TBase::Become(&TConsoleRequestActor::MainState);
+
+        if (const auto timeout = TDuration::MilliSeconds(Request.GetTimeoutMs())) {
+            ctx.Schedule(timeout, new TEvents::TEvWakeup());
+        }
     }
 
     void SendRequest(const TActorContext &ctx)
@@ -328,6 +332,10 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
         SendReplyAndDie(ctx);
     }
 
+    void HandleTimeout(const TActorContext &ctx) {
+        ReplyWithErrorAndDie(Ydb::StatusIds::TIMEOUT, "Console request timed out", ctx);
+    }
+
     STFUNC(MainState) {
         switch (ev->GetTypeRewrite()) {
             CFunc(TEvents::TEvUndelivered::EventType, Undelivered);
@@ -347,6 +355,7 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
             HFunc(TEvConsole::TEvToggleConfigValidatorResponse, Handle);
             CFunc(TEvTabletPipe::EvClientDestroyed, Undelivered);
             HFunc(TEvTabletPipe::TEvClientConnected, Handle);
+            SFunc(TEvents::TEvWakeup, HandleTimeout);
         default:
             Y_ABORT("TConsoleRequestActor::MainState unexpected event type: %" PRIx32 " event: %s",
                    ev->GetTypeRewrite(),

ydb/core/client/server/msgbus_server_console.cpp

@@ -1417,10 +1417,13 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
             FilterDomainKey[TSubDomainKey(domainInfo->DomainKey.OwnerId, domainInfo->DomainKey.LocalPathId)] = path;
 
             TTabletId hiveId = domainInfo->Params.GetHive();
-            if (hiveId && NeedToAskHive(hiveId)) {
+            if (hiveId) {
                 DatabaseState[path].HiveId = hiveId;
-                AskHive(path, hiveId);
+                if (NeedToAskHive(hiveId)) {
+                    AskHive(path, hiveId);
+                }
             } else if (RootHiveId && NeedToAskHive(RootHiveId)) {
+                DatabaseState[DomainPath].HiveId = RootHiveId;
                 AskHive(DomainPath, RootHiveId);
             }
 
@@ -1515,23 +1518,31 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
 
     void AggregateHiveInfo() {
         TNodeTabletState::TTabletStateSettings settings;
-        for (const auto& [hiveId, hiveResponse] : HiveInfo) {
+        for (auto& [dbPath, dbState] : DatabaseState) {
+            const auto& hiveResponse = HiveInfo[dbState.HiveId];
             if (hiveResponse.IsOk()) {
                 settings.AliveBarrier = TInstant::MilliSeconds(hiveResponse->Record.GetResponseTimestamp()) - TDuration::Minutes(5);
                 settings.MaxRestartsPerPeriod = HealthCheckConfig.GetThresholds().GetTabletsRestartsOrange();
                 for (const NKikimrHive::TTabletInfo& hiveTablet : hiveResponse->Record.GetTablets()) {
                     TSubDomainKey tenantId = TSubDomainKey(hiveTablet.GetObjectDomain());
                     auto itDomain = FilterDomainKey.find(tenantId);
+                    TDatabaseState* database = nullptr;
                     if (itDomain == FilterDomainKey.end()) {
-                        continue;
-                    }
-                    auto itDatabase = DatabaseState.find(itDomain->second);
-                    if (itDatabase == DatabaseState.end()) {
-                        continue;
+                        if (!FilterDatabase || FilterDatabase == dbPath) {
+                            database = &dbState;
+                        } else {
+                            continue;
+                        }
+                    } else {
+                        auto itDatabase = DatabaseState.find(itDomain->second);
+                        if (itDatabase != DatabaseState.end()) {
+                            database = &itDatabase->second;
+                        } else {
+                            continue;
+                        }
                     }
-                    TDatabaseState& database = itDatabase->second;
                     auto tabletId = std::make_pair(hiveTablet.GetTabletID(), hiveTablet.GetFollowerID());
-                    database.MergedTabletState.emplace(tabletId, &hiveTablet);
+                    database->MergedTabletState.emplace(tabletId, &hiveTablet);
                     TNodeId nodeId = hiveTablet.GetNodeID();
                     switch (hiveTablet.GetVolatileState()) {
                         case NKikimrHive::ETabletVolatileState::TABLET_VOLATILE_STATE_STARTING:
@@ -1541,7 +1552,7 @@ class TSelfCheckRequest : public TActorBootstrapped<TSelfCheckRequest> {
                             nodeId = 0;
                             break;
                     }
-                    database.MergedNodeTabletState[nodeId].AddTablet(hiveTablet, settings);
+                    database->MergedNodeTabletState[nodeId].AddTablet(hiveTablet, settings);
                 }
             }
         }

ydb/core/health_check/health_check.cpp

@@ -446,6 +446,15 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         UNIT_ASSERT_VALUES_EQUAL(issueVdiscCount, issueVdiscNumber);
     }
 
+    bool HasTabletIssue(const Ydb::Monitoring::SelfCheckResult& result) {
+       for (const auto& issue_log : result.issue_log()) {
+            if (issue_log.level() == 4 && issue_log.type() == "TABLET") {
+                return true;
+            }
+        }
+        return false;
+    }
+
     void ListingTest(int const groupNumber, int const vdiscPerGroupNumber, bool const isMergeRecords = false) {
         auto result = RequestHc(groupNumber, vdiscPerGroupNumber, isMergeRecords);
         CheckHcResult(result, groupNumber, vdiscPerGroupNumber, isMergeRecords);
@@ -865,6 +874,15 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         }
     }
 
+    void AddBadServerlessTablet(TEvHive::TEvResponseHiveInfo::TPtr* ev) {
+        auto &record = (*ev)->Get()->Record;
+        auto* tablet = record.MutableTablets()->Add();
+        tablet->SetTabletID(1);
+        tablet->MutableObjectDomain()->SetSchemeShard(SERVERLESS_DOMAIN_KEY.OwnerId);
+        tablet->MutableObjectDomain()->SetPathId(SERVERLESS_DOMAIN_KEY.LocalPathId);
+        tablet->SetRestartsPerPeriod(500);
+    }
+
     Y_UNIT_TEST(SpecificServerless) {
         TPortManager tp;
         ui16 port = tp.GetPort(2134);
@@ -1164,6 +1182,102 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         UNIT_ASSERT(!databaseFoundInResult);
     }
 
+    Y_UNIT_TEST(ServerlessBadTablets) {
+        TPortManager tp;
+        ui16 port = tp.GetPort(2134);
+        ui16 grpcPort = tp.GetPort(2135);
+        auto settings = TServerSettings(port)
+                .SetNodeCount(1)
+                .SetDynamicNodeCount(1)
+                .SetUseRealThreads(false)
+                .SetDomainName("Root");
+        TServer server(settings);
+        server.EnableGRpc(grpcPort);
+        TClient client(settings);
+        TTestActorRuntime& runtime = *server.GetRuntime();
+
+        auto &dynamicNameserviceConfig = runtime.GetAppData().DynamicNameserviceConfig;
+        dynamicNameserviceConfig->MaxStaticNodeId = runtime.GetNodeId(server.StaticNodes() - 1);
+        dynamicNameserviceConfig->MinDynamicNodeId = runtime.GetNodeId(server.StaticNodes());
+        dynamicNameserviceConfig->MaxDynamicNodeId = runtime.GetNodeId(server.StaticNodes() + server.DynamicNodes() - 1);
+
+        ui32 sharedDynNodeId = runtime.GetNodeId(1);
+
+        bool firstConsoleResponse = true;
+        auto observerFunc = [&](TAutoPtr<IEventHandle>& ev) {
+            switch (ev->GetTypeRewrite()) {
+                 case NConsole::TEvConsole::EvListTenantsResponse: {
+                    auto *x = reinterpret_cast<NConsole::TEvConsole::TEvListTenantsResponse::TPtr*>(&ev);
+                    AddPathsToListTenantsResponse(x, { "/Root/serverless", "/Root/shared" });
+                    break;
+                }
+                case NConsole::TEvConsole::EvGetTenantStatusResponse: {
+                    auto *x = reinterpret_cast<NConsole::TEvConsole::TEvGetTenantStatusResponse::TPtr*>(&ev);
+                    if (!firstConsoleResponse) {
+                        ChangeGetTenantStatusResponse(x, "/Root/serverless");
+                    } else {
+                        firstConsoleResponse = false;
+                        ChangeGetTenantStatusResponse(x, "/Root/shared");
+                    }
+                    break;
+                }
+                case TEvTxProxySchemeCache::EvNavigateKeySetResult: {
+                    auto *x = reinterpret_cast<TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr*>(&ev);
+                    ChangeNavigateKeyResultServerless(x, NKikimrSubDomains::EServerlessComputeResourcesModeShared, runtime);
+                    break;
+                }
+                case TEvHive::EvResponseHiveNodeStats: {
+                    auto *x = reinterpret_cast<TEvHive::TEvResponseHiveNodeStats::TPtr*>(&ev);
+                    ChangeResponseHiveNodeStats(x, sharedDynNodeId);
+                    break;
+                }
+                case TEvHive::EvResponseHiveInfo: {
+                    auto *x = reinterpret_cast<TEvHive::TEvResponseHiveInfo::TPtr*>(&ev);
+                    AddBadServerlessTablet(x);
+                    break;
+                }
+                case TEvSchemeShard::EvDescribeSchemeResult: {
+                    auto *x = reinterpret_cast<NSchemeShard::TEvSchemeShard::TEvDescribeSchemeResult::TPtr*>(&ev);
+                    ChangeDescribeSchemeResultServerless(x);
+                    break;
+                }
+                case TEvBlobStorage::EvControllerConfigResponse: {
+                    auto *x = reinterpret_cast<TEvBlobStorage::TEvControllerConfigResponse::TPtr*>(&ev);
+                    AddGroupVSlotInControllerConfigResponseWithStaticGroup(x, NKikimrBlobStorage::TGroupStatus::FULL, TVDisks(1));
+                    break;
+                }
+                case NSysView::TEvSysView::EvGetVSlotsResponse: {
+                    auto* x = reinterpret_cast<NSysView::TEvSysView::TEvGetVSlotsResponse::TPtr*>(&ev);
+                    AddVSlotsToSysViewResponse(x, 1, TVDisks(1));
+                    break;
+                }
+                case NSysView::TEvSysView::EvGetGroupsResponse: {
+                    auto* x = reinterpret_cast<NSysView::TEvSysView::TEvGetGroupsResponse::TPtr*>(&ev);
+                    AddGroupsToSysViewResponse(x);
+                    break;
+                }
+                case NSysView::TEvSysView::EvGetStoragePoolsResponse: {
+                    auto* x = reinterpret_cast<NSysView::TEvSysView::TEvGetStoragePoolsResponse::TPtr*>(&ev);
+                    AddStoragePoolsToSysViewResponse(x);
+                    break;
+                }
+            }
+
+            return TTestActorRuntime::EEventAction::PROCESS;
+        };
+        runtime.SetObserverFunc(observerFunc);
+
+        TActorId sender = runtime.AllocateEdgeActor();
+        TAutoPtr<IEventHandle> handle;
+
+        auto *request = new NHealthCheck::TEvSelfCheckRequest;
+        request->Request.set_return_verbose_status(true);
+        runtime.Send(new IEventHandle(NHealthCheck::MakeHealthCheckID(), sender, request, 0));
+        const auto result = runtime.GrabEdgeEvent<NHealthCheck::TEvSelfCheckResult>(handle)->Result;
+        Ctest << result.ShortDebugString();
+        UNIT_ASSERT(HasTabletIssue(result));
+    }
+
     Y_UNIT_TEST(DontIgnoreServerlessWithExclusiveNodesWhenNotSpecific) {
         TPortManager tp;
         ui16 port = tp.GetPort(2134);
@@ -1859,15 +1973,6 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         ShardsQuotaTest(105, 0, 0, Ydb::Monitoring::StatusFlag::GREEN);
     }
 
-    bool HasDeadTabletIssue(const Ydb::Monitoring::SelfCheckResult& result) {
-       for (const auto& issue_log : result.issue_log()) {
-            if (issue_log.level() == 4 && issue_log.type() == "TABLET") {
-                return true;
-            }
-        }
-        return false;
-    }
-
     Y_UNIT_TEST(TestTabletIsDead) {
         TPortManager tp;
         ui16 port = tp.GetPort(2134);
@@ -1895,7 +2000,7 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         auto result = runtime->GrabEdgeEvent<NHealthCheck::TEvSelfCheckResult>(handle)->Result;
         Cerr << result.ShortDebugString();
 
-        UNIT_ASSERT(HasDeadTabletIssue(result));
+        UNIT_ASSERT(HasTabletIssue(result));
     }
 
     Y_UNIT_TEST(TestBootingTabletIsNotDead) {
@@ -1926,7 +2031,7 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         auto result = runtime->GrabEdgeEvent<NHealthCheck::TEvSelfCheckResult>(handle)->Result;
         Cerr << result.ShortDebugString();
 
-        UNIT_ASSERT(!HasDeadTabletIssue(result));
+        UNIT_ASSERT(!HasTabletIssue(result));
     }
 
     Y_UNIT_TEST(TestReBootingTabletIsDead) {
@@ -1960,7 +2065,7 @@ Y_UNIT_TEST_SUITE(THealthCheckTest) {
         auto result = runtime->GrabEdgeEvent<NHealthCheck::TEvSelfCheckResult>(handle)->Result;
         Cerr << result.ShortDebugString();
 
-        UNIT_ASSERT(HasDeadTabletIssue(result));
+        UNIT_ASSERT(HasTabletIssue(result));
     }
 
     void SendHealthCheckConfigUpdate(TTestActorRuntime &runtime, const TActorId& sender, const NKikimrConfig::THealthCheckConfig &cfg) {

ydb/core/health_check/health_check_ut.cpp

@@ -103,7 +103,7 @@ class TKafkaConnection: public TActorBootstrapped<TKafkaConnection>, public TNet
 
     void Bootstrap() {
         Context->ConnectionId = SelfId();
-        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement;
+        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement || NKikimr::AppData()->PQConfig.GetRequireCredentialsInNewProtocol();
         // if no authentication required, then we can use local database as our target
         if (!Context->RequireAuthentication) {
             Context->DatabasePath = NKikimr::AppData()->TenantName;
@@ -478,7 +478,7 @@ class TKafkaConnection: public TActorBootstrapped<TKafkaConnection>, public TNet
             return;
         }
 
-        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement;
+        Context->RequireAuthentication = NKikimr::AppData()->EnforceUserTokenRequirement || NKikimr::AppData()->PQConfig.GetRequireCredentialsInNewProtocol();
         Context->UserToken = event->UserToken;
         Context->DatabasePath = event->DatabasePath;
         Context->AuthenticationStep = authStep;

ydb/core/kafka_proxy/kafka_connection.cpp

@@ -17,20 +17,54 @@ void TBootQueue::AddToBootQueue(TBootQueueRecord record) {
 }
 
 TBootQueue::TBootQueueRecord TBootQueue::PopFromBootQueue() {
-    TBootQueueRecord record = BootQueue.top();
-    BootQueue.pop();
+    TQueue& currentQueue = GetCurrentQueue();
+    TBootQueueRecord record = currentQueue.top();
+    currentQueue.pop();
+    if (ProcessWaitQueue) {
+        NextFromWaitQueue = !NextFromWaitQueue;
+    }
     return record;
 }
 
 void TBootQueue::AddToWaitQueue(TBootQueueRecord record) {
-    WaitQueue.emplace_back(record);
+    WaitQueue.push(record);
+}
+
+void TBootQueue::IncludeWaitQueue() {
+    ProcessWaitQueue = true;
 }
 
-void TBootQueue::MoveFromWaitQueueToBootQueue() {
-    for (TBootQueueRecord record : WaitQueue) {
-        AddToBootQueue(record);
+void TBootQueue::ExcludeWaitQueue() {
+    ProcessWaitQueue = false;
+}
+
+bool TBootQueue::Empty() const {
+    if (ProcessWaitQueue) {
+        return BootQueue.empty() && WaitQueue.empty();
+    } else {
+        return BootQueue.empty();
+    }
+}
+
+size_t TBootQueue::Size() const {
+    if (ProcessWaitQueue) {
+        return BootQueue.size() + WaitQueue.size();
+    } else {
+        return BootQueue.size();
+    }
+}
+
+TBootQueue::TQueue& TBootQueue::GetCurrentQueue() {
+    if (BootQueue.empty()) {
+        return WaitQueue;
+    }
+    if (WaitQueue.empty()) {
+        return BootQueue;
+    }
+    if (ProcessWaitQueue && NextFromWaitQueue) {
+        return WaitQueue;
     }
-    WaitQueue.clear();
+    return BootQueue;
 }
 
 }