Add jitter and exponential delay to configs request (#19299)

Author: Enjection

ydb/core/config/init/init.cpp

@@ -399,28 +399,61 @@ class TDefaultDynConfigClient
         std::shared_ptr<IConfigurationResult> res;
         bool success = false;
         TString error;
-
         SetRandomSeed(TInstant::Now().MicroSeconds());
-        int minAttempts = 10;
-        int attempts = 0;
-        while (!success && attempts < minAttempts) {
+
+        const int maxRounds = 10;
+        const TDuration baseRoundDelay = TDuration::MilliSeconds(500);
+        const TDuration maxIntraAddrDelay = TDuration::Minutes(3);
+        const TDuration maxDelay = TDuration::Minutes(5);
+        const TDuration baseAddressDelay = TDuration::MilliSeconds(250);
+
+        auto sleepWithJitteredExponentialDelay = [&env](TDuration baseDelay, TDuration maxDelay, int exponent) {
+            ui64 multiplier = 1ULL << exponent;
+            TDuration delay = baseDelay * multiplier;
+            delay = Min(delay, maxDelay);
+            
+            ui64 maxMs = delay.MilliSeconds();
+            ui64 jitteredMs = RandomNumber<ui64>(maxMs + 1);
+            TDuration jitteredDelay = TDuration::MilliSeconds(jitteredMs);
+            
+            env.Sleep(jitteredDelay);
+        };
+
+        int round = 0;
+        int totalAttempts = 0;
+
+        while (!success && round < maxRounds) {
+            int addressIndex = 0;
             for (auto addr : addrs) {
+                // internal timeout is 5 seconds
                 success = TryToLoadConfigForDynamicNodeFromCMS(grpcSettings, addr, settings, env, logger, res, error);
-                ++attempts;
+                ++totalAttempts;
+                
                 if (success) {
                     break;
                 }
+                
+                // Exponential delay between individual addresses - delay grows with each address in the round
+                if (addrs.size() > 1) {
+                    sleepWithJitteredExponentialDelay(baseAddressDelay, maxIntraAddrDelay, Max(addressIndex, round));
+                }
+                
+                ++addressIndex;
             }
-            // Randomized backoff
+            
             if (!success) {
-                env.Sleep(TDuration::MilliSeconds(500 + RandomNumber<ui64>(1000)));
-            } else {
-                break;
+                ++round;
+                
+                if (round < maxRounds) {
+                    sleepWithJitteredExponentialDelay(baseRoundDelay, maxDelay, round - 1);
+                }
             }
         }
 
         if (!success) {
-            logger.Err() << "WARNING: couldn't load config from CMS: " << error << Endl;
+            logger.Err() << "WARNING: couldn't load config from Console after " 
+                        << totalAttempts << " attempts across " << round 
+                        << " rounds: " << error << Endl;
         }
 
         return res;

ydb/tests/functional/ydb_cli/test_ydb_backup.py

@@ -1319,7 +1319,7 @@ def setup_class(cls):
             storage_pool_units_count={
                 'hdd': 1
             },
-            timeout_seconds=100,
+            timeout_seconds=240,
             token=cls.cluster.config.default_clusteradmin
         )
 
@@ -1527,7 +1527,7 @@ def restore(cls, command, input, additional_args=[], token=""):
                 "--endpoint", f"grpc://localhost:{cls.restore_cluster.nodes[1].grpc_port}",
             ]
             + command
-            + ["--input", backup_files_dir, "-w", "60s"]
+            + ["--input", backup_files_dir, "-w", "240s"]
             + additional_args,
             env={"YDB_TOKEN": token}
         )

ydb/tests/functional/ydb_cli/ya.make

@@ -37,5 +37,7 @@ PEERDIR(
 )
 
 FORK_TEST_FILES()
+FORK_SUBTESTS()
+SPLIT_FACTOR(30)
 
 END()

ydb/tests/library/harness/kikimr_cluster_interface.py

@@ -153,7 +153,7 @@ def __wait_tenant_up(
             self,
             database_name,
             expected_computational_units=None,
-            timeout_seconds=120,
+            timeout_seconds=240,
             token=None
     ):
         def predicate():