Fix quorum for distconf when in Bridge mode (#19366)

Author: alexvru

ydb/core/base/blobstorage.h

@@ -1368,6 +1368,13 @@ struct TEvBlobStorage {
             , GroupId(groupId)
         {}
 
+        TEvGetResult(NKikimrProto::EReplyStatus status, ui32 sz, TArrayHolder<TResponse> responses, TGroupId groupId)
+            : Status(status)
+            , ResponseSz(sz)
+            , Responses(std::move(responses))
+            , GroupId(groupId.GetRawId())
+        {}
+
         TString Print(bool isFull) const {
             TStringStream str;
             str << "TEvGetResult {Status# " << NKikimrProto::EReplyStatus_Name(Status).data();
@@ -2463,6 +2470,12 @@ struct TEvBlobStorage {
             , StatusFlags(statusFlags)
         {}
 
+        TEvStatusResult(NKikimrProto::EReplyStatus status, TStorageStatusFlags statusFlags, float approximateFreeSpaceShare)
+            : Status(status)
+            , StatusFlags(statusFlags)
+            , ApproximateFreeSpaceShare(approximateFreeSpaceShare)
+        {}
+
         TString Print(bool isFull) const {
             Y_UNUSED(isFull);
             TStringStream str;

ydb/core/base/bridge.h

@@ -23,6 +23,11 @@ namespace NKikimr {
         const TPile *BeingPromotedPile = nullptr; // a reference to the pile being promoted, or nullptr if none are promoted
 
         using TPtr = std::shared_ptr<const TBridgeInfo>;
+
+        const TPile *GetPile(TBridgePileId bridgePileId) const {
+            Y_ABORT_UNLESS(bridgePileId.GetRawId() < Piles.size());
+            return &Piles[bridgePileId.GetRawId()];
+        }
     };
 
 } // NKikimr

ydb/core/base/id_wrapper.h

@@ -64,7 +64,7 @@ template <typename T, typename Tag> class TIdWrapper {
         TIdWrapper old = *this;
         operator++();
         return old;
-  }
+    }
 
     friend std::ostream &operator<<(std::ostream &out, TIdWrapper &id) {
         return out << id.Raw;
@@ -74,7 +74,7 @@ template <typename T, typename Tag> class TIdWrapper {
         return out << id.Raw;
     }
 
-    constexpr auto operator<=>(const TIdWrapper &) const = default;
+    friend std::strong_ordering operator<=>(const TIdWrapper&, const TIdWrapper&) = default;
 
     T GetRawId() const { return Raw; }
 
@@ -83,14 +83,16 @@ template <typename T, typename Tag> class TIdWrapper {
     friend THash<TIdWrapper<T, Tag>>;
 };
 
-template <typename T, typename Tag> struct std::hash<TIdWrapper<T, Tag>> {
+template<typename T, typename Tag>
+struct std::hash<TIdWrapper<T, Tag>> {
     std::size_t operator()(const TIdWrapper<T, Tag> &id) const {
         return std::hash<T>{}(id.Raw);
     }
 };
 
-template <typename T, typename Tag> struct THash<TIdWrapper<T, Tag>> {
+template<typename T, typename Tag>
+struct THash<TIdWrapper<T, Tag>> {
     std::size_t operator()(const TIdWrapper<T, Tag> &id) const {
-        return THash<T>()(id.Raw);
+        return THash<T>{}(id.Raw);
     }
 };

ydb/core/blobstorage/dsproxy/bridge/bridge.cpp

@@ -10,15 +10,22 @@
 namespace NKikimr::NStorage {
 
     struct TNodeIdentifier : std::tuple<TString, ui32, ui32> {
+        std::optional<TBridgePileId> BridgePileId;
+
         TNodeIdentifier() = default;
 
-        TNodeIdentifier(TString host, ui32 port, ui32 nodeId)
+        TNodeIdentifier(TString host, ui32 port, ui32 nodeId, std::optional<TBridgePileId> bridgePileId)
             : std::tuple<TString, ui32, ui32>(std::move(host), port, nodeId)
+            , BridgePileId(bridgePileId)
         {}
 
         TNodeIdentifier(const NKikimrBlobStorage::TNodeIdentifier& proto)
             : std::tuple<TString, ui32, ui32>(proto.GetHost(), proto.GetPort(), proto.GetNodeId())
-        {}
+        {
+            if (proto.HasBridgePileId()) {
+                BridgePileId.emplace(TBridgePileId::FromProto(&proto, &NKikimrBlobStorage::TNodeIdentifier::GetBridgePileId));
+            }
+        }
 
         ui32 NodeId() const {
             return std::get<2>(*this);
@@ -28,6 +35,9 @@ namespace NKikimr::NStorage {
             proto->SetHost(std::get<0>(*this));
             proto->SetPort(std::get<1>(*this));
             proto->SetNodeId(std::get<2>(*this));
+            if (BridgePileId) {
+                BridgePileId->CopyToProto(proto, &NKikimrBlobStorage::TNodeIdentifier::SetBridgePileId);
+            }
         }
     };
 
@@ -563,207 +573,6 @@ namespace NKikimr::NStorage {
         }
     }
 
-    template<typename T>
-    bool HasDiskQuorum(const NKikimrBlobStorage::TStorageConfig& config, T&& generateSuccessful) {
-        // generate set of all required drives
-        THashMap<TString, std::tuple<ui32, ui32>> status; // dc -> {ok, err}
-        THashMap<ui32, const NKikimrBlobStorage::TNodeIdentifier*> nodeMap;
-        THashSet<std::tuple<TNodeIdentifier, TString>> allDrives;
-        auto cb = [&status, &allDrives](const auto& node, const auto& drive) {
-            auto& [ok, err] = status[TNodeLocation(node.GetLocation()).GetDataCenterId()];
-            ++err;
-            allDrives.emplace(node, drive.GetPath());
-        };
-        EnumerateConfigDrives(config, 0, cb, &nodeMap);
-
-        // process responses
-        generateSuccessful([&](const TNodeIdentifier& node, const TString& path, std::optional<ui64> /*guid*/) {
-            const auto it = nodeMap.find(node.NodeId());
-            if (it == nodeMap.end() || TNodeIdentifier(*it->second) != node) { // unexpected node answers
-                return;
-            }
-            if (!allDrives.erase(std::make_tuple(node, path))) { // unexpected drive
-                return;
-            }
-            auto& [ok, err] = status[TNodeLocation(it->second->GetLocation()).GetDataCenterId()];
-            Y_ABORT_UNLESS(err);
-            ++ok;
-            --err;
-        });
-
-        // calculate number of good and bad datacenters
-        ui32 ok = 0;
-        ui32 err = 0;
-        for (const auto& [_, value] : status) {
-            const auto [dcOk, dcErr] = value;
-            ++(dcOk > dcErr ? ok : err);
-        }
-
-        // strict datacenter majority
-        return ok > err;
-    }
-
-    template<typename T>
-    bool HasNodeQuorum(const NKikimrBlobStorage::TStorageConfig& config, T&& generateSuccessful) {
-        // generate set of all nodes
-        THashMap<TString, std::tuple<ui32, ui32>> status; // dc -> {ok, err}
-        THashMap<ui32, const NKikimrBlobStorage::TNodeIdentifier*> nodeMap;
-        for (const auto& node : config.GetAllNodes()) {
-            auto& [ok, err] = status[TNodeLocation(node.GetLocation()).GetDataCenterId()];
-            ++err;
-            nodeMap.emplace(node.GetNodeId(), &node);
-        }
-
-        // process responses
-        std::set<TNodeIdentifier> seen;
-        generateSuccessful([&](const TNodeIdentifier& node) {
-            const auto& [_, inserted] = seen.insert(node);
-            Y_ABORT_UNLESS(inserted);
-
-            const auto it = nodeMap.find(node.NodeId());
-            if (it == nodeMap.end() || TNodeIdentifier(*it->second) != node) { // unexpected node answers
-                return;
-            }
-            auto& [ok, err] = status[TNodeLocation(it->second->GetLocation()).GetDataCenterId()];
-            Y_ABORT_UNLESS(err);
-            ++ok;
-            --err;
-        });
-
-        // calculate number of good and bad datacenters
-        ui32 ok = 0;
-        ui32 err = 0;
-        for (const auto& [_, value] : status) {
-            const auto [dcOk, dcErr] = value;
-            ++(dcOk > dcErr ? ok : err);
-        }
-
-        // strict datacenter majority
-        return ok > err;
-    }
-
-    template<typename T>
-    bool HasStorageQuorum(const NKikimrBlobStorage::TStorageConfig& config, T&& generateSuccessful,
-            const TNodeWardenConfig& nwConfig, bool allowUnformatted) {
-        auto makeError = [&](TString error) -> bool {
-            STLOG(PRI_CRIT, BS_NODE, NWDC41, "configuration incorrect", (Error, error));
-            Y_DEBUG_ABORT("%s", error.c_str());
-            return false;
-        };
-        if (!config.HasBlobStorageConfig()) { // no storage config at all -- however, this is quite strange
-            return makeError("no BlobStorageConfig section in config");
-        }
-        const auto& bsConfig = config.GetBlobStorageConfig();
-        if (!bsConfig.HasServiceSet()) { // maybe this is initial configuration
-            return !config.GetGeneration() || makeError("non-initial configuration with missing ServiceSet");
-        }
-        const auto& ss = bsConfig.GetServiceSet();
-
-        // build map of group infos
-        struct TGroupRecord {
-            TIntrusivePtr<TBlobStorageGroupInfo> Info;
-            TBlobStorageGroupInfo::TGroupVDisks Confirmed; // a set of confirmed group disks
-
-            TGroupRecord(TIntrusivePtr<TBlobStorageGroupInfo>&& info)
-                : Info(std::move(info))
-                , Confirmed(&Info->GetTopology())
-            {}
-        };
-        THashMap<ui32, TGroupRecord> groups;
-        for (const auto& group : ss.GetGroups()) {
-            const ui32 groupId = group.GetGroupID();
-            if (TGroupID(groupId).ConfigurationType() != EGroupConfigurationType::Static) {
-                return makeError("nonstatic group id in static configuration section");
-            }
-
-            TStringStream err;
-            TIntrusivePtr<TBlobStorageGroupInfo> info = TBlobStorageGroupInfo::Parse(group, &nwConfig.StaticKey, &err);
-            if (!info) {
-                return makeError(TStringBuilder() << "failed to parse static group " << groupId << ": " << err.Str());
-            }
-
-            if (const auto [it, inserted] = groups.emplace(groupId, std::move(info)); !inserted) {
-                return makeError("duplicate group id in static configuration section");
-            }
-        }
-
-        // fill in pdisk map
-        THashMap<std::tuple<ui32, ui32, ui64>, TString> pdiskIdToPath; // (nodeId, pdiskId, pdiskGuid) -> path
-        for (const auto& pdisk : ss.GetPDisks()) {
-            const auto [it, inserted] = pdiskIdToPath.emplace(std::make_tuple(pdisk.GetNodeID(), pdisk.GetPDiskID(),
-                pdisk.GetPDiskGuid()), pdisk.GetPath());
-            if (!inserted) {
-                return makeError("duplicate pdisk in static configuration section");
-            }
-        }
-
-        // create confirmation map
-        THashMultiMap<std::tuple<ui32, TString, std::optional<ui64>>, TVDiskID> confirm;
-        for (const auto& vdisk : ss.GetVDisks()) {
-            if (!vdisk.HasVDiskID() || !vdisk.HasVDiskLocation()) {
-                return makeError("incorrect TVDisk record");
-            }
-            if (vdisk.GetEntityStatus() == NKikimrBlobStorage::EEntityStatus::DESTROY) {
-                continue;
-            }
-            if (vdisk.HasDonorMode()) {
-                continue;
-            }
-            const auto vdiskId = VDiskIDFromVDiskID(vdisk.GetVDiskID());
-            const auto it = groups.find(vdiskId.GroupID.GetRawId());
-            if (it == groups.end()) {
-                return makeError(TStringBuilder() << "VDisk " << vdiskId << " does not match any static group");
-            }
-            const TGroupRecord& group = it->second;
-            if (vdiskId.GroupGeneration != group.Info->GroupGeneration) {
-                return makeError(TStringBuilder() << "VDisk " << vdiskId << " group generation mismatch");
-            }
-            const auto& location = vdisk.GetVDiskLocation();
-            const auto jt = pdiskIdToPath.find(std::make_tuple(location.GetNodeID(), location.GetPDiskID(),
-                location.GetPDiskGuid()));
-            if (jt == pdiskIdToPath.end()) {
-                return makeError(TStringBuilder() << "VDisk " << vdiskId << " points to incorrect PDisk record");
-            }
-            confirm.emplace(std::make_tuple(location.GetNodeID(), jt->second, location.GetPDiskGuid()), vdiskId);
-            if (allowUnformatted) {
-                confirm.emplace(std::make_tuple(location.GetNodeID(), jt->second, std::nullopt), vdiskId);
-            }
-        }
-
-        // process responded nodes
-        generateSuccessful([&](const TNodeIdentifier& node, const TString& path, std::optional<ui64> guid) {
-            const auto key = std::make_tuple(node.NodeId(), path, guid);
-            const auto [begin, end] = confirm.equal_range(key);
-            for (auto it = begin; it != end; ++it) {
-                const TVDiskID& vdiskId = it->second;
-                TGroupRecord& group = groups.at(vdiskId.GroupID.GetRawId());
-                group.Confirmed |= {&group.Info->GetTopology(), vdiskId};
-            }
-        });
-
-        // scan all groups and find ones without quorum
-        for (const auto& [groupId, group] : groups) {
-            if (group.Info->IsBridged()) {
-                continue;
-            }
-            if (const auto& checker = group.Info->GetQuorumChecker(); !checker.CheckQuorumForGroup(group.Confirmed)) {
-                return false;
-            }
-        }
-
-        return true; // all group meet their quorums
-    }
-
-    // Ensure configuration has quorum in both disk and storage ways for current and previous configuration.
-    template<typename T>
-    bool HasConfigQuorum(const NKikimrBlobStorage::TStorageConfig& config, T&& generateSuccessful,
-            const TNodeWardenConfig& nwConfig, bool mindPrev = true) {
-        return HasDiskQuorum(config, generateSuccessful) &&
-            HasStorageQuorum(config, generateSuccessful, nwConfig, true) && (!mindPrev || !config.HasPrevConfig() || (
-            HasDiskQuorum(config.GetPrevConfig(), generateSuccessful) &&
-            HasStorageQuorum(config.GetPrevConfig(), generateSuccessful, nwConfig, false)));
-    }
-
     std::optional<TString> ValidateConfigUpdate(const NKikimrBlobStorage::TStorageConfig& current,
             const NKikimrBlobStorage::TStorageConfig& proposed);
 

ydb/core/blobstorage/nodewarden/distconf.h

@@ -10,7 +10,7 @@ namespace NKikimr::NStorage {
         const ui32 selfNodeId = SelfId().NodeId();
         for (const auto& item : ev->Get()->Nodes) {
             if (item.NodeId == selfNodeId) {
-                SelfNode = TNodeIdentifier(item.ResolveHost, item.Port, selfNodeId);
+                SelfNode = TNodeIdentifier(item.ResolveHost, item.Port, selfNodeId, std::nullopt);
                 Y_ABORT_UNLESS(IsSelfStatic == item.IsStatic);
             }
             if (item.IsStatic) {

ydb/core/blobstorage/nodewarden/distconf_binding.cpp

@@ -1,4 +1,5 @@
 #include "distconf.h"
+#include "distconf_quorum.h"
 
 namespace NKikimr::NStorage {