25-1: Fix KesusQuoterService reconnect (#20758)

Author: CyberROFL

ydb/core/quoter/kesus_quoter_proxy.cpp

@@ -43,6 +43,8 @@ namespace NQuoter {
 
 namespace TEvKesus = NKesus::TEvKesus;
 
+const ui64 KesusReconnectLimit = 5;
+
 class TKesusQuoterProxy : public TActorBootstrapped<TKesusQuoterProxy> {
     struct TResourceState {
         const TString Resource;
@@ -797,15 +799,18 @@ class TKesusQuoterProxy : public TActorBootstrapped<TKesusQuoterProxy> {
         if (ev->Get()->Status == NKikimrProto::OK) {
             KESUS_PROXY_LOG_DEBUG("Successfully connected to tablet");
             Connected = true;
+            KesusReconnectCount = 0;
             SubscribeToAllResources();
         } else {
             if (ev->Get()->Dead) {
                 KESUS_PROXY_LOG_WARN("Tablet doesn't exist");
                 SendToService(CreateUpdateEvent(TEvQuota::EUpdateState::Broken));
             } else {
                 KESUS_PROXY_LOG_WARN("Failed to connect to tablet. Status: " << ev->Get()->Status);
-                if (!ConnectToKesus(true)) {
-                    KESUS_PROXY_LOG_WARN("Too many reconnect attempts, assuming kesus dead");
+                if (++KesusReconnectCount <= KesusReconnectLimit) {
+                    ConnectToKesus(true);
+                } else {
+                    KESUS_PROXY_LOG_WARN("Too many reconnect attempts in a row, assuming kesus dead");
                     SendToService(CreateUpdateEvent(TEvQuota::EUpdateState::Broken));
                     KesusReconnectCount = 0;
                 }
@@ -1122,7 +1127,7 @@ class TKesusQuoterProxy : public TActorBootstrapped<TKesusQuoterProxy> {
         return KesusInfo->Description.GetKesusTabletId();
     }
 
-    NTabletPipe::TClientConfig GetPipeConnectionOptions(bool reconnection) {
+    static NTabletPipe::TClientConfig GetPipeConnectionOptions(bool reconnection) {
         NTabletPipe::TClientConfig cfg;
         cfg.RetryPolicy = {
             .RetryLimitCount = 3u,
@@ -1138,13 +1143,9 @@ class TKesusQuoterProxy : public TActorBootstrapped<TKesusQuoterProxy> {
         CookieToResourcePath.clear(); // we will resend all requests with new cookies
     }
 
-    bool ConnectToKesus(bool reconnection) {
+    void ConnectToKesus(bool reconnection) {
         if (reconnection) {
             KESUS_PROXY_LOG_INFO("Reconnecting to kesus");
-            ++KesusReconnectCount;
-            if (KesusReconnectCount > 5) {
-                return false;
-            }
         } else {
             KESUS_PROXY_LOG_DEBUG("Connecting to kesus");
         }
@@ -1157,7 +1158,6 @@ class TKesusQuoterProxy : public TActorBootstrapped<TKesusQuoterProxy> {
                     GetKesusTabletId(),
                     GetPipeConnectionOptions(reconnection)));
         Connected = false;
-        return true;
     }
 
     void PassAway() override {

ydb/core/quoter/kesus_quoter_proxy.h

@@ -22,6 +22,8 @@ struct ITabletPipeFactory {
     static THolder<ITabletPipeFactory> GetDefaultFactory();
 };
 
+extern const ui64 KesusReconnectLimit;
+
 NActors::IActor* CreateKesusQuoterProxy(ui64 quoterId, const NSchemeCache::TSchemeCacheNavigate::TEntry& navEntry, const NActors::TActorId& quoterServiceId, THolder<ITabletPipeFactory> tabletPipeFactory = ITabletPipeFactory::GetDefaultFactory());
 
 class TKesusResourceAllocationStatistics {

ydb/core/quoter/kesus_quoter_ut.cpp

@@ -257,6 +257,23 @@ Y_UNIT_TEST_SUITE(KesusProxyTest) {
         setup.GetRuntime().SimulateSleep(TDuration::Zero());
     }
 
+    Y_UNIT_TEST(ReconnectsWithKesusAfterSeveralRetries) {
+        TKesusProxyTestSetup setup;
+        auto* pipeMock = setup.GetPipeFactory().ExpectTabletPipeCreation(true);
+        EXPECT_CALL(*pipeMock, OnPoisonPill());
+
+        for (ui32 i = 1; i <= NQuoter::KesusReconnectLimit; ++i) {
+            setup.SendNotConnected(pipeMock);
+            setup.WaitConnected();
+        }
+
+        setup.SendDestroyed(pipeMock);
+        setup.WaitPipesCreated(1 /* initial */ + NQuoter::KesusReconnectLimit + 1 /* last */);
+
+        // Dispatch some events to let poison pill reach the mock
+        setup.GetRuntime().SimulateSleep(TDuration::Zero());
+    }
+
     Y_UNIT_TEST(RejectsNotCanonizedResourceName) {
         TKesusProxyTestSetup setup;