Refactor harness (#9211)

Author: maximyurchuk

ydb/tests/functional/encryption/test_encryption.py

@@ -74,7 +74,7 @@ def setup_class(cls):
             )
         )
         cls.cluster.start()
-        cls.discovery_endpoint = "%s:%s" % (cls.cluster.nodes[1].hostname, cls.cluster.nodes[1].grpc_port)
+        cls.discovery_endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].grpc_port)
 
     def test_simple_encryption(self):
         databases = []

ydb/tests/functional/query_cache/test_query_cache.py

@@ -28,7 +28,7 @@ def setup_class(cls):
         cls.config.yaml_config["table_service_config"] = {"compile_query_cache_size": QUERY_CACHE_SIZE}
         cls.cluster = kikimr_cluster_factory(configurator=cls.config)
         cls.cluster.start()
-        cls.discovery_endpoint = "%s:%s" % (cls.cluster.nodes[1].hostname, cls.cluster.nodes[1].grpc_port)
+        cls.discovery_endpoint = "%s:%s" % (cls.cluster.nodes[1].host, cls.cluster.nodes[1].grpc_port)
         cls.driver = ydb.Driver(endpoint=cls.discovery_endpoint, database="/Root", credentials=ydb.AnonymousCredentials())
         cls.driver.wait(timeout=5)
         cls.pool = ydb.SessionPool(cls.driver)

ydb/tests/library/harness/daemon.py

@@ -17,7 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
-def extract_stderr_details(stderr_file, max_lines=0):
+def _extract_stderr_details(stderr_file, max_lines=0):
     if max_lines == 0:
         return []
 
@@ -43,7 +43,7 @@ def __init__(self, message, stdout, stderr, exit_code, max_stderr_lines=0):
                     "Stdout file name: \n{}".format(stdout if stdout is not None else "is not present."),
                     "Stderr file name: \n{}".format(stderr if stderr is not None else "is not present."),
                 ]
-                + extract_stderr_details(stderr, max_stderr_lines)
+                + _extract_stderr_details(stderr, max_stderr_lines)
             )
         )
 
@@ -54,6 +54,7 @@ def __init__(self, exceptions):
 
 
 class Daemon(object):
+    """Local process executed as process in current host"""
     def __init__(
         self,
         command,
@@ -208,6 +209,7 @@ def kill(self):
 
 @six.add_metaclass(abc.ABCMeta)
 class ExternalNodeDaemon(object):
+    """External daemon, executed as process in separate host, managed via ssh"""
     def __init__(self, host):
         self._host = host
         self._ssh_username = param_constants.ssh_username
@@ -263,12 +265,6 @@ def cleanup_logs(self):
             'sudo rm -rf {}/* && sudo service rsyslog restart'.format(self.logs_directory), raise_on_error=True
         )
 
-    def sky_get_and_move(self, rb_torrent, item_to_move, target_path):
-        self.ssh_command(['sky get -d %s %s' % (self._artifacts_path, rb_torrent)], raise_on_error=True)
-        self.ssh_command(
-            ['sudo mv %s %s' % (os.path.join(self._artifacts_path, item_to_move), target_path)], raise_on_error=True
-        )
-
     def send_signal(self, signal):
         self.ssh_command(
             "ps aux | grep %d | grep -v daemon | grep -v grep | awk '{ print $2 }' | xargs sudo kill -%d"

ydb/tests/library/harness/kikimr_config.py

@@ -33,7 +33,7 @@
 ])
 
 
-def get_fqdn():
+def _get_fqdn():
     hostname = socket.gethostname()
     addrinfo = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, 0, 0, socket.AI_CANONNAME)
     for ai in addrinfo:
@@ -44,7 +44,7 @@ def get_fqdn():
 
 
 # GRPC_SERVER:DEBUG,TICKET_PARSER:WARN,KQP_COMPILE_ACTOR:DEBUG
-def get_additional_log_configs():
+def _get_additional_log_configs():
     log_configs = os.getenv('YDB_ADDITIONAL_LOG_CONFIGS', '')
     rt = {}
 
@@ -55,13 +55,13 @@ def get_additional_log_configs():
     return rt
 
 
-def get_grpc_host():
+def _get_grpc_host():
     if sys.platform == "darwin":
         return "localhost"
     return "[::]"
 
 
-def load_default_yaml(default_tablet_node_ids, ydb_domain_name, static_erasure, log_configs):
+def _load_default_yaml(default_tablet_node_ids, ydb_domain_name, static_erasure, log_configs):
     data = read_binary(__name__, "resources/default_yaml.yml")
     if isinstance(data, bytes):
         data = data.decode('utf-8')
@@ -73,7 +73,7 @@ def load_default_yaml(default_tablet_node_ids, ydb_domain_name, static_erasure,
         ydb_default_log_level=int(LogLevels.from_string(os.getenv("YDB_DEFAULT_LOG_LEVEL", "NOTICE"))),
         ydb_domain_name=ydb_domain_name,
         ydb_static_erasure=static_erasure,
-        ydb_grpc_host=get_grpc_host(),
+        ydb_grpc_host=_get_grpc_host(),
         ydb_pq_topics_are_first_class_citizen=bool(os.getenv("YDB_PQ_TOPICS_ARE_FIRST_CLASS_CITIZEN", "true")),
         ydb_pq_cluster_table_path=str(os.getenv("YDB_PQ_CLUSTER_TABLE_PATH", "")),
         ydb_pq_version_table_path=str(os.getenv("YDB_PQ_VERSION_TABLE_PATH", "")),
@@ -97,7 +97,7 @@ def _load_yaml_config(filename):
     return yaml.safe_load(_read_file(filename))
 
 
-def use_in_memory_pdisks_var(pdisk_store_path, use_in_memory_pdisks):
+def _use_in_memory_pdisks_var(pdisk_store_path, use_in_memory_pdisks):
     if os.getenv('YDB_USE_IN_MEMORY_PDISKS') is not None:
         return os.getenv('YDB_USE_IN_MEMORY_PDISKS') == "true"
 
@@ -120,9 +120,7 @@ def __init__(
             output_path=None,
             enable_pq=True,
             pq_client_service_types=None,
-            slot_count=0,
             pdisk_store_path=None,
-            version=None,
             enable_sqs=False,
             domain_name='Root',
             suppress_version_check=True,
@@ -168,7 +166,6 @@ def __init__(
         if extra_grpc_services is None:
             extra_grpc_services = []
 
-        self._version = version
         self.use_log_files = use_log_files
         self.suppress_version_check = suppress_version_check
         self._pdisk_store_path = pdisk_store_path
@@ -184,7 +181,7 @@ def __init__(
         self._pdisks_info = []
         if self.__grpc_ssl_enable:
             self.__grpc_tls_data_path = grpc_tls_data_path or yatest_common.output_path()
-            cert_pem, key_pem = tls_tools.generate_selfsigned_cert(get_fqdn())
+            cert_pem, key_pem = tls_tools.generate_selfsigned_cert(_get_fqdn())
             self.__grpc_tls_ca = cert_pem
             self.__grpc_tls_key = key_pem
             self.__grpc_tls_cert = cert_pem
@@ -204,20 +201,19 @@ def __init__(
         self.state_storage_rings = state_storage_rings
         if self.state_storage_rings is None:
             self.state_storage_rings = copy.deepcopy(self.__node_ids[: 9 if erasure == Erasure.MIRROR_3_DC else 8])
-        self.__use_in_memory_pdisks = use_in_memory_pdisks_var(pdisk_store_path, use_in_memory_pdisks)
+        self.__use_in_memory_pdisks = _use_in_memory_pdisks_var(pdisk_store_path, use_in_memory_pdisks)
         self.__pdisks_directory = os.getenv('YDB_PDISKS_DIRECTORY')
         self.static_erasure = erasure
         self.domain_name = domain_name
         self.__number_of_pdisks_per_node = 1 + len(dynamic_pdisks)
         self.__load_udfs = load_udfs
         self.__udfs_path = udfs_path
-        self.__slot_count = slot_count
         self._dcs = [1]
         if erasure == Erasure.MIRROR_3_DC:
             self._dcs = [1, 2, 3]
 
         self.__additional_log_configs = {} if additional_log_configs is None else additional_log_configs
-        self.__additional_log_configs.update(get_additional_log_configs())
+        self.__additional_log_configs.update(_get_additional_log_configs())
         if pg_compatible_expirement:
             self.__additional_log_configs.update({
                 'PGWIRE': LogLevels.from_string('DEBUG'),
@@ -236,7 +232,7 @@ def __init__(
 
         self.__bs_cache_file_path = bs_cache_file_path
 
-        self.yaml_config = load_default_yaml(self.__node_ids, self.domain_name, self.static_erasure, self.__additional_log_configs)
+        self.yaml_config = _load_default_yaml(self.__node_ids, self.domain_name, self.static_erasure, self.__additional_log_configs)
 
         if overrided_actor_system_config:
             self.yaml_config["actor_system_config"] = overrided_actor_system_config

ydb/tests/library/harness/kikimr_runner.py

@@ -215,10 +215,6 @@ def send_signal(self, signal):
     def host(self):
         return 'localhost'
 
-    @property
-    def hostname(self):
-        return kikimr_config.get_fqdn()
-
     @property
     def port(self):
         return self.grpc_port
@@ -380,19 +376,16 @@ def __register_node(self):
         )
         return self._nodes[node_index]
 
-    def register_slots(self, database, count=1, encryption_key=None):
-        return [self.register_slot(database, encryption_key) for _ in range(count)]
+    def __register_slots(self, database, count=1, encryption_key=None):
+        return [self.__register_slot(database, encryption_key) for _ in range(count)]
 
     def register_and_start_slots(self, database, count=1, encryption_key=None):
-        slots = self.register_slots(database, count, encryption_key)
+        slots = self.__register_slots(database, count, encryption_key)
         for slot in slots:
             slot.start()
         return slots
 
-    def register_slot(self, tenant_affiliation=None, encryption_key=None):
-        return self._register_slot(tenant_affiliation, encryption_key)
-
-    def _register_slot(self, tenant_affiliation=None, encryption_key=None):
+    def __register_slot(self, tenant_affiliation=None, encryption_key=None):
         slot_index = next(self._slot_index_allocator)
         node_broker_port = (
             self.nodes[1].grpc_ssl_port if self.__configurator.grpc_ssl_enable
@@ -412,12 +405,12 @@ def _register_slot(self, tenant_affiliation=None, encryption_key=None):
         )
         return self._slots[slot_index]
 
-    def unregister_slots(self, slots):
+    def __unregister_slots(self, slots):
         for i in slots:
             del self._slots[i.node_id]
 
     def unregister_and_stop_slots(self, slots):
-        self.unregister_slots(slots)
+        self.__unregister_slots(slots)
         for i in slots:
             i.stop()