check if nodes restart while benchmark (#9166)

Author: iddqdex

ydb/tests/olap/lib/ydb_cli.py

@@ -82,6 +82,8 @@ def _try_extract_error_message(stderr: str) -> str:
             if wait_error is not None:
                 return YdbCliHelper.WorkloadRunResult(error_message=f'Ydb cluster is dead: {wait_error}')
 
+            cluster_start_time = YdbCluster.get_cluster_info().get('max_start_time', 0)
+
             json_path = yatest.common.work_path(f'q{query_num}.json')
             qout_path = yatest.common.work_path(f'q{query_num}.out')
             plan_path = yatest.common.work_path(f'q{query_num}.plan')
@@ -111,6 +113,8 @@ def _try_extract_error_message(stderr: str) -> str:
                         err = f'Invalid return code: {exec.returncode} instesd 0.'
             except (yatest.common.process.TimeoutError, yatest.common.process.ExecutionTimeoutError):
                 err = f'Timeout {timeout}s expeared.'
+            if YdbCluster.get_cluster_info().get('max_start_time', 0) != cluster_start_time:
+                err = ('' if err is None else f'{err}\n\n') + 'Some nodes were restart'
             stats = {}
             if (os.path.exists(json_path)):
                 with open(json_path, 'r') as r:

ydb/tests/olap/lib/ydb_cluster.py

@@ -52,9 +52,11 @@ def get_cluster_info(cls):
             version = ''
             cluster_name = ''
             nodes_wilcard = ''
+            max_start_time = 0
             nodes, node_count = cls._get_cluster_nodes()
             for node in nodes:
                 n = node.get('SystemState', {})
+                max_start_time = max(max_start_time, int(n.get('StartTime', 0)))
                 cluster_name = n.get('ClusterName', cluster_name)
                 version = n.get('Version', version)
                 for tenant in n.get('Tenants', []):
@@ -67,6 +69,7 @@ def get_cluster_info(cls):
                 'name': cluster_name,
                 'nodes_wilcard': nodes_wilcard,
                 'service_url': cls._get_service_url(),
+                'max_start_time': max_start_time,
             }
         return deepcopy(cls._cluster_info)