make nodes less critical (to make cluster less critical) (#19677)

Author: adameat

ydb/core/tablet/node_whiteboard.cpp

@@ -767,72 +767,59 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
         }
     }
 
-    void UpdateSystemState(const TActorContext &ctx) {
+    void UpdateSystemState() {
         NKikimrWhiteboard::EFlag eFlag = NKikimrWhiteboard::EFlag::Green;
-        NKikimrWhiteboard::EFlag pDiskFlag = NKikimrWhiteboard::EFlag::Green;
-        ui32 yellowFlags = 0;
+        ui32 badDisks = 0;
         double maxDiskUsage = 0;
         for (const auto& pr : PDiskStateInfo) {
-            if (!pr.second.HasState()) {
-                pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Yellow);
-                ++yellowFlags;
-            } else {
+            if (pr.second.HasState()) {
                 switch (pr.second.GetState()) {
                 case NKikimrBlobStorage::TPDiskState::InitialFormatReadError:
                 case NKikimrBlobStorage::TPDiskState::InitialSysLogReadError:
                 case NKikimrBlobStorage::TPDiskState::InitialSysLogParseError:
                 case NKikimrBlobStorage::TPDiskState::InitialCommonLogReadError:
                 case NKikimrBlobStorage::TPDiskState::InitialCommonLogParseError:
                 case NKikimrBlobStorage::TPDiskState::CommonLoggerInitError:
-                    pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Red);
-                    break;
                 case NKikimrBlobStorage::TPDiskState::OpenFileError:
-                    pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Yellow);
-                    ++yellowFlags;
+                    eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Yellow);
+                    ++badDisks;
                     break;
                 default:
                     break;
                 }
             }
             if (pr.second.HasAvailableSize() && pr.second.GetTotalSize() != 0) {
                 double avail = (double)pr.second.GetAvailableSize() / pr.second.GetTotalSize();
-                if (avail <= 0.06) {
-                    pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Red);
+                if (avail <= 0.04) {
+                    eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Orange);
                 } else if (avail <= 0.08) {
-                    pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Orange);
-                } else if (avail <= 0.15) {
-                    pDiskFlag = std::max(pDiskFlag, NKikimrWhiteboard::EFlag::Yellow);
-                    ++yellowFlags;
+                    eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Yellow);
                 }
                 maxDiskUsage = std::max(maxDiskUsage, 1.0 - avail);
             }
         }
         if (PDiskStateInfo.size() > 0) {
             SystemStateInfo.SetMaxDiskUsage(maxDiskUsage);
         }
-        if (pDiskFlag == NKikimrWhiteboard::EFlag::Yellow) {
-            switch (yellowFlags) {
-            case 1:
-                break;
-            case 2:
-                pDiskFlag = NKikimrWhiteboard::EFlag::Orange;
-                break;
-            case 3:
-                pDiskFlag = NKikimrWhiteboard::EFlag::Red;
-                break;
-            }
+        if (eFlag == NKikimrWhiteboard::EFlag::Yellow && badDisks > 1) {
+            eFlag = NKikimrWhiteboard::EFlag::Orange;
         }
-        eFlag = std::max(eFlag, pDiskFlag);
         for (const auto& pr : VDiskStateInfo) {
             eFlag = std::max(eFlag, pr.second.GetDiskSpace());
-            eFlag = std::max(eFlag, pr.second.GetSatisfactionRank().GetFreshRank().GetFlag());
-            eFlag = std::max(eFlag, pr.second.GetSatisfactionRank().GetLevelRank().GetFlag());
-        }
-        if (SystemStateInfo.HasMessageBusState()) {
-            eFlag = std::max(eFlag, SystemStateInfo.GetMessageBusState());
+            if (pr.second.GetDiskSpace() >= NKikimrWhiteboard::EFlag::Red) {
+                eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Orange);
+            } else if (pr.second.GetDiskSpace() > NKikimrWhiteboard::EFlag::Green) {
+                eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Yellow);
+            }
+            if (pr.second.GetSatisfactionRank().GetFreshRank().GetFlag() > NKikimrWhiteboard::EFlag::Green) {
+                eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Yellow);
+            }
+            if (pr.second.GetSatisfactionRank().GetLevelRank().GetFlag() > NKikimrWhiteboard::EFlag::Green) {
+                eFlag = std::max(eFlag, NKikimrWhiteboard::EFlag::Yellow);
+            }
         }
         if (SystemStateInfo.HasGRpcState()) {
-            eFlag = std::max(eFlag, SystemStateInfo.GetGRpcState());
+            eFlag = std::max(eFlag, std::max(SystemStateInfo.GetGRpcState(), NKikimrWhiteboard::EFlag::Orange));
         }
         for (const auto& stats : SystemStateInfo.GetPoolStats()) {
             double usage = stats.GetUsage();
@@ -846,11 +833,18 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
             } else  {
                 flag = NKikimrWhiteboard::EFlag::Green;
             }
-            eFlag = Max(eFlag, flag);
+            if (stats.GetName() == "User") {
+                flag = std::min(flag, NKikimrWhiteboard::EFlag::Orange);
+            } else if (stats.GetName() == "IO") {
+                flag = std::min(flag, NKikimrWhiteboard::EFlag::Yellow);
+            } else if (stats.GetName() == "Batch") {
+                flag = std::min(flag, NKikimrWhiteboard::EFlag::Green);
+            }
+            eFlag = std::max(eFlag, flag);
         }
         if (!SystemStateInfo.HasSystemState() || SystemStateInfo.GetSystemState() != eFlag) {
             SystemStateInfo.SetSystemState(eFlag);
-            SystemStateInfo.SetChangeTime(ctx.Now().MilliSeconds());
+            SystemStateInfo.SetChangeTime(TActivationContext::Now().MilliSeconds());
         }
     }
 
@@ -1158,7 +1152,7 @@ class TNodeWhiteboardService : public TActorBootstrapped<TNodeWhiteboardService>
                 threadInfo->MutableStates()->emplace(state.first, state.second);
             }
         }
-        UpdateSystemState(ctx);
+        UpdateSystemState();
         ctx.Schedule(UPDATE_PERIOD, new TEvPrivate::TEvUpdateRuntimeStats());
     }