Bridge mode in Scheme Board: ignore disconnected ring groups in populator (#19884)

Co-authored-by: Ilnaz Nizametdinov <i.nizametdinov@gmail.com>

Author: jepett0

ydb/core/testlib/basics/helpers.h

@@ -40,7 +40,7 @@ namespace NFake {
 
     void SetupStateStorage(TTestActorRuntime& runtime, ui32 nodeIndex,
                            bool replicasOnFirstNode = false);
-    void SetupCustomStateStorage(TTestActorRuntime &runtime, ui32 NToSelect, ui32 nrings, ui32 ringSize); 
+    void SetupCustomStateStorage(TTestActorRuntime &runtime, ui32 NToSelect, ui32 nrings, ui32 ringSize);
     void SetupBSNodeWarden(TTestActorRuntime& runtime, ui32 nodeIndex, TIntrusivePtr<TNodeWardenConfig> nodeWardenConfig);
     void SetupTabletResolver(TTestActorRuntime& runtime, ui32 nodeIndex);
     void SetupTabletPipePerNodeCaches(TTestActorRuntime& runtime, ui32 nodeIndex, bool forceFollowers = false);
@@ -76,4 +76,6 @@ namespace NFake {
         {}
     };
 
+    TActorId MakeBoardReplicaID(ui32 node, ui32 replicaIndex);
+
 }

ydb/core/testlib/basics/services.cpp

@@ -192,7 +192,7 @@ namespace NPDisk {
     }
 
     static TIntrusivePtr<TStateStorageInfo> GenerateStateStorageInfo(const TVector<TActorId> &replicas, ui32 NToSelect, ui32 nrings, ui32 ringSize)
-    {   
+    {
         Y_ABORT_UNLESS(replicas.size() >= nrings * ringSize);
         Y_ABORT_UNLESS(NToSelect <= nrings);
 
@@ -201,7 +201,7 @@ namespace NPDisk {
         auto& group = info->RingGroups.back();
         group.NToSelect = NToSelect;
         group.Rings.resize(nrings);
-            
+
         ui32 inode = 0;
         for (size_t i = 0; i < nrings; ++i) {
             for (size_t j = 0; j < ringSize; ++j) {
@@ -212,10 +212,7 @@ namespace NPDisk {
         return info;
     }
 
-    static TActorId MakeBoardReplicaID(
-        const ui32 node,
-        const ui32 replicaIndex
-    ) {
+    TActorId MakeBoardReplicaID(ui32 node, ui32 replicaIndex) {
         char x[12] = { 's', 's', 'b' };
         x[3] = (char)1;
         memcpy(x + 5, &replicaIndex, sizeof(ui32));
@@ -224,10 +221,10 @@ namespace NPDisk {
 
     void SetupCustomStateStorage(
         TTestActorRuntime &runtime,
-        ui32 NToSelect, 
-        ui32 nrings, 
+        ui32 NToSelect,
+        ui32 nrings,
         ui32 ringSize)
-    {   
+    {
         TVector<TActorId> ssreplicas;
         for (size_t i = 0; i < nrings * ringSize; ++i) {
             ssreplicas.push_back(MakeStateStorageReplicaID(runtime.GetNodeId(i), i));
@@ -249,7 +246,7 @@ namespace NPDisk {
         auto sbInfo = GenerateStateStorageInfo(sbreplicas, NToSelect, nrings, ringSize);
         auto bInfo = GenerateStateStorageInfo(breplicas, NToSelect, nrings, ringSize);
 
-        
+
         for (ui32 ssIndex = 0; ssIndex < nrings * ringSize; ++ssIndex) {
             runtime.AddLocalService(ssreplicas[ssIndex],
                 TActorSetupCmd(CreateStateStorageReplica(ssInfo.Get(), ssIndex), TMailboxType::Revolving, 0), ssIndex);
@@ -265,7 +262,7 @@ namespace NPDisk {
         }
     }
 
-    
+
     void SetupStateStorage(TTestActorRuntime& runtime, ui32 nodeIndex, bool firstNode)
     {
         const TActorId ssreplicas[3] = {

ydb/core/tx/scheme_board/populator.cpp

@@ -50,6 +50,14 @@ namespace {
 
 } // anonymous
 
+bool ShouldIgnore(const TStateStorageInfo::TRingGroup& ringGroup) {
+    return ringGroup.WriteOnly || ringGroup.State == ERingGroupState::DISCONNECTED;
+}
+
+bool IsMajorityReached(const TStateStorageInfo::TRingGroup& ringGroup, ui32 ringGroupAcks) {
+    return ringGroupAcks > ringGroup.NToSelect / 2;
+}
+
 class TReplicaPopulator: public TMonitorableActor<TReplicaPopulator> {
     void ProcessSync(NInternalEvents::TEvDescribeResult* msg = nullptr, const TPathId& pathId = TPathId()) {
         if (msg == nullptr) {
@@ -707,36 +715,31 @@ class TPopulator: public TMonitorableActor<TPopulator> {
         }
     }
 
-    bool CheckQuorum(TVector<ui32> &replicaRepliesCounter, TActorId foundReplica) const {
-        ui32 quorumCnt = 0;
-        for (const auto& ringGroup : GroupInfo->RingGroups) {
-            if (!ringGroup.WriteOnly) {
-                ++quorumCnt;
-            }
-        }
-        TVector<bool> quorum(GroupInfo->RingGroups.size());
-
+    void ProcessReplicaAck(TVector<ui32>& ringGroupAcks, TActorId ackedReplica, TVector<bool>& ringGroupQuorums) const {
         for (ui32 ringGroupIndex : xrange(GroupInfo->RingGroups.size())) {
             const auto& ringGroup = GroupInfo->RingGroups[ringGroupIndex];
-            if (ringGroup.WriteOnly) {
-                continue;
-            }
             for (const auto& ring : ringGroup.Rings) {
                 for (const auto& replica : ring.Replicas) {
-                    if (replica == foundReplica 
-                        && ++replicaRepliesCounter[ringGroupIndex] > (ringGroup.NToSelect / 2) 
-                        && !quorum[ringGroupIndex])
-                    {
-                        quorum[ringGroupIndex] = true;
-                        --quorumCnt;
-                        if (quorumCnt == 0) {
-                            return true;
+                    if (replica == ackedReplica) {
+                        ++ringGroupAcks[ringGroupIndex];
+                        if (IsMajorityReached(ringGroup, ringGroupAcks[ringGroupIndex])) {
+                            ringGroupQuorums[ringGroupIndex] = true;
                         }
+                        break;
                     }
                 }
             }
         }
-        return quorumCnt == 0;
+    }
+
+    bool CheckQuorum(TVector<ui32>& ringGroupAcks, TActorId ackedReplica) const {
+        TVector<bool> ringGroupQuorums(GroupInfo->RingGroups.size(), false);
+        for (ui32 ringGroupIndex : xrange(GroupInfo->RingGroups.size())) {
+            const auto& ringGroup = GroupInfo->RingGroups[ringGroupIndex];
+            ringGroupQuorums[ringGroupIndex] = ShouldIgnore(ringGroup) || IsMajorityReached(ringGroup, ringGroupAcks[ringGroupIndex]);
+        }
+        ProcessReplicaAck(ringGroupAcks, ackedReplica, ringGroupQuorums);
+        return Count(ringGroupQuorums, false) == 0;
     }
 
     void Handle(NSchemeshardEvents::TEvUpdateAck::TPtr& ev) {
@@ -761,9 +764,9 @@ class TPopulator: public TMonitorableActor<TPopulator> {
         while (pathIt != it->second.PathAcks.end()
                && pathIt->first.first == pathId
                && pathIt->first.second <= version) {
-            TActorId* foundReplica = ReplicaToReplicaPopulatorBackMap.FindPtr(ev->Sender);
-            Y_ABORT_UNLESS(foundReplica != nullptr);
-            if (CheckQuorum(pathIt->second, *foundReplica)) {
+            TActorId* ackedReplica = ReplicaToReplicaPopulatorBackMap.FindPtr(ev->Sender);
+            Y_ABORT_UNLESS(ackedReplica != nullptr);
+            if (CheckQuorum(pathIt->second, *ackedReplica)) {
                 SBP_LOG_N("Ack update"
                     << ": ack to# " << it->second.AckTo
                     << ", cookie# " << ev->Cookie

ydb/core/tx/scheme_board/populator.h

@@ -3,12 +3,21 @@
 #include "defs.h"
 #include "two_part_description.h"
 
+#include <ydb/core/base/statestorage.h>
 #include <ydb/core/scheme/scheme_pathid.h>
 
 #include <util/generic/map.h>
 
 namespace NKikimr {
 
+namespace NSchemeBoard {
+
+    bool ShouldIgnore(const TStateStorageInfo::TRingGroup& ringGroup);
+
+    bool IsMajorityReached(const TStateStorageInfo::TRingGroup& ringGroup, ui32 ringGroupAcks);
+
+}
+
 IActor* CreateSchemeBoardPopulator(
     const ui64 owner,
     const ui64 generation,