Feature/ydbd slice/allow slice creation using raw config yaml (#12762)

YdbWorkloadTopic.Full_Statistics_UseTx is muted now

Author: shmel1k

ydb/tools/cfg/base.py

@@ -264,7 +264,7 @@ def normalize_domain(domain_name):
 
 
 class ClusterDetailsProvider(object):
-    def __init__(self, template, walle_provider, validator=None, database=None):
+    def __init__(self, template, walle_provider, validator=None, database=None, use_new_style_cfg=False):
         if not validator:
             validator = validation.default_validator()
 
@@ -286,7 +286,7 @@ def __init__(self, template, walle_provider, validator=None, database=None):
         self.__racks = {}
         self.__bodies = {}
         self.__dcs = {}
-        self.use_new_style_kikimr_cfg = self.__cluster_description.get("use_new_style_kikimr_cfg", False)
+        self.use_new_style_kikimr_cfg = self.__cluster_description.get("use_new_style_kikimr_cfg", use_new_style_cfg)
         self.need_generate_app_config = self.__cluster_description.get("need_generate_app_config", False)
         self.need_txt_files = self.__cluster_description.get("need_txt_files", True)
         self.use_auto_config = self.__cluster_description.get("use_auto_config", False)
@@ -297,6 +297,7 @@ def __init__(self, template, walle_provider, validator=None, database=None):
         self.table_profiles_config = self.__cluster_description.get("table_profiles_config")
         self.http_proxy_config = self.__cluster_description.get("http_proxy_config")
         self.blob_storage_config = self.__cluster_description.get("blob_storage_config")
+        self.channel_profile_config = self.__cluster_description.get("channel_profile_config")
         self.pdisk_key_config = self.__cluster_description.get("pdisk_key_config", {})
         if not self.need_txt_files and not self.use_new_style_kikimr_cfg:
             assert "cannot remove txt files without new style kikimr cfg!"
@@ -353,16 +354,25 @@ def forbid_implicit_storage_pools(self):
     def _get_datacenter(self, host_description):
         if host_description.get("datacenter") is not None:
             return str(host_description.get("datacenter"))
+        dc = host_description.get("location", {}).get("data_center", None)
+        if dc:
+            return str(dc)
         return str(self._walle.get_datacenter(host_description.get("name", host_description.get("host"))))
 
     def _get_rack(self, host_description):
         if host_description.get("rack") is not None:
             return str(host_description.get("rack"))
+        rack = host_description.get("location", {}).get("rack", None)
+        if rack:
+            return str(rack)
         return str(self._walle.get_rack(host_description.get("name", host_description.get("host"))))
 
     def _get_body(self, host_description):
         if host_description.get("body") is not None:
             return str(host_description.get("body"))
+        body = host_description.get("location", {}).get("body", None)
+        if body:
+            return str(body)
         return str(self._walle.get_body(host_description.get("name", host_description.get("host"))))
 
     def _collect_drives_info(self, host_description):
@@ -555,14 +565,16 @@ def domains(self):
             domain_name = domain.get("domain_name")
 
             storage_pool_kinds = {
-                pool_kind.get("kind"): self.__storage_pool_kind(pool_kind) for pool_kind in domain.get("storage_pool_kinds", [])
+                pool_kind.get("kind"): self.__storage_pool_kind(pool_kind)
+                for pool_kind in domain.get("storage_pool_kinds", [])
             }
             assert len(set(storage_pool_kinds.keys())) == len(
                 storage_pool_kinds.keys()
             ), "required unique kind value in storage_pool_kinds items"
 
             storage_pools = [
-                self.__storage_pool(storage_pool_kinds, pool_instance, domain_name) for pool_instance in domain.get("storage_pools", [])
+                self.__storage_pool(storage_pool_kinds, pool_instance, domain_name)
+                for pool_instance in domain.get("storage_pools", [])
             ]
 
             domains.append(
@@ -604,7 +616,9 @@ def nw_cache_file_path(self):
 
     @property
     def fail_domain_type(self):
-        return types.FailDomainType.from_string(str(self.__cluster_description.get("fail_domain_type", DEFAULT_FAIL_DOMAIN_TYPE)))
+        return types.FailDomainType.from_string(
+            str(self.__cluster_description.get("fail_domain_type", DEFAULT_FAIL_DOMAIN_TYPE))
+        )
 
     @property
     def min_fail_domains(self):

ydb/tools/cfg/static.py

@@ -510,10 +510,14 @@ def get_normalized_config(self):
                             if 'pdisk_config' in vdisk_location:
                                 if 'expected_slot_count' in vdisk_location['pdisk_config']:
                                     vdisk_location['pdisk_config']['expected_slot_count'] = int(vdisk_location['pdisk_config']['expected_slot_count'])
-        if 'channel_profile_config' in normalized_config:
-            for profile in normalized_config['channel_profile_config']['profile']:
-                for channel in profile['channel']:
-                    channel['pdisk_category'] = int(channel['pdisk_category'])
+        if self.__cluster_details.channel_profile_config is not None:
+            normalized_config["channel_profile_config"] = self.__cluster_details.channel_profile_config
+        else:
+            if 'channel_profile_config' in normalized_config:
+                for profile in normalized_config['channel_profile_config']['profile']:
+                    for channel in profile['channel']:
+                        print(channel)
+                        channel['pdisk_category'] = int(channel['pdisk_category'])
         if 'system_tablets' in normalized_config:
             for tablets in normalized_config['system_tablets'].values():
                 for tablet in tablets:
@@ -754,6 +758,8 @@ def __generate_bs_txt(self):
         dc_enumeration = {}
 
         if not self.__cluster_details.get_service("static_groups"):
+            if self.__cluster_details.blob_storage_config:
+                return
             self.__proto_configs["bs.txt"] = self._read_generated_bs_config(
                 str(self.__cluster_details.static_erasure),
                 str(self.__cluster_details.min_fail_domains),
@@ -835,13 +841,17 @@ def __generate_bs_txt(self):
         if self.__cluster_details.nw_cache_file_path is not None:
             self.__proto_configs["bs.txt"].CacheFilePath = self.__cluster_details.nw_cache_file_path
 
-    def _read_generated_bs_config(self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config):
+    def _read_generated_bs_config(
+        self, static_erasure, min_fail_domains, static_pdisk_type, fail_domain_type, bs_format_config
+    ):
         result = config_pb2.TBlobStorageConfig()
 
-        with tempfile.NamedTemporaryFile(delete=True) as t_file:
+        with tempfile.NamedTemporaryFile(delete=False) as t_file:
             utils.write_proto_to_file(t_file.name, bs_format_config)
 
-            rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[types.FailDomainType.from_string(fail_domain_type)]
+            rx_begin, rx_end, dx_begin, dx_end = types.DistinctionLevels[
+                types.FailDomainType.from_string(fail_domain_type)
+            ]
 
             cmd_base = [
                 self.__local_binary_path,

ydb/tools/ydbd_slice/__init__.py

@@ -2,12 +2,14 @@
 import sys
 import json
 import signal
-import shutil
 import tempfile
 import logging
 import argparse
 import subprocess
 import warnings
+
+import library.python.resource as rs
+
 from urllib3.exceptions import HTTPWarning
 
 from ydb.tools.cfg.walle import NopHostsInformationProvider
@@ -499,7 +501,42 @@ def ssh_args():
         metavar="SSH_USER",
         default=current_user,
         help="user for ssh interaction with slice. Default value is $USER "
-             "(which equals {user} now)".format(user=current_user),
+        "(which equals {user} now)".format(user=current_user),
+    )
+    return args
+
+
+def databases_config_path_args():
+    args = argparse.ArgumentParser(add_help=False)
+    args.add_argument(
+        "--databases-config",
+        metavar="DATABASES_CONFIG",
+        default="",
+        required=False,
+        help="Path to file with databases configuration",
+    )
+    return args
+
+
+def cluster_type_args():
+    args = argparse.ArgumentParser(add_help=False)
+    args.add_argument(
+        "--cluster-type",
+        metavar="CLUSTER_TYPE",
+        required=True,
+        help="Erasure type for slice",
+        choices=["block-4-2-8-nodes", "mirror-3-dc-3-nodes-in-memory", "mirror-3-dc-3-nodes", "mirror-3-dc-9-nodes"],
+    )
+    return args
+
+
+def output_file():
+    args = argparse.ArgumentParser(add_help=False)
+    args.add_argument(
+        "--output-file",
+        metavar="OUTPUT_FILE",
+        required=False,
+        help="File to save cluster configuration",
     )
     return args
 
@@ -583,7 +620,7 @@ def dispatch_run(func, args, walle_provider):
 
     if clear_tmp:
         logger.debug("remove temp dirs '%s'", temp_dir)
-        shutil.rmtree(temp_dir)
+        # shutil.rmtree(temp_dir)
 
 
 def add_install_mode(modes, walle_provider):
@@ -593,9 +630,17 @@ def _run(args):
     mode = modes.add_parser(
         "install",
         conflict_handler='resolve',
-        parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), log_args(), ssh_args()],
+        parents=[
+            direct_nodes_args(),
+            cluster_description_args(),
+            binaries_args(),
+            component_args(),
+            log_args(),
+            ssh_args(),
+            # databases_config_path_args(),
+        ],
         description="Full installation of the cluster from scratch. "
-                    "You can use --hosts to specify particular hosts. But it is tricky."
+        "You can use --hosts to specify particular hosts. But it is tricky.",
     )
     mode.set_defaults(handler=_run)
 
@@ -672,7 +717,7 @@ def _run(args):
         "clear",
         parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
         description="Stop all kikimr instances at the nodes, format all kikimr drivers, shutdown dynamic slots. "
-                    "And don't start nodes afrer it. "
+                    "And don't start nodes after it. "
                     "Use --hosts to specify particular hosts."
     )
     mode.set_defaults(handler=_run)
@@ -686,12 +731,42 @@ def _run(args):
         "format",
         parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
         description="Stop all kikimr instances at the nodes, format all kikimr drivers at the nodes, start the instances. "
-                    "If you call format for all cluster, you will spoil it. "
-                    "Additional dynamic configuration will required after it. "
-                    "If you call format for few nodes, cluster will regenerate after it. "
-                    "Use --hosts to specify particular hosts."
+        "If you call format for all cluster, you will spoil it. "
+        "Additional dynamic configuration will required after it. "
+        "If you call format for few nodes, cluster will regenerate after it. "
+        "Use --hosts to specify particular hosts.",
+    )
+    mode.set_defaults(handler=_run)
+
 
+def add_sample_config_mode(modes):
+    def _run(args):
+        cluster_type = args.cluster_type
+        template_path = ""
+        if cluster_type == "block-4-2-8-nodes":
+            template_path = "/ydbd_slice/baremetal/templates/block-4-2-8-nodes.yaml"
+        elif cluster_type == "mirror-3-dc-3-nodes-in-memory":
+            pass
+        elif cluster_type == "mirror-3-dc-3-nodes":
+            template_path = "/ydbd_slice/baremetal/templates/mirror-3-dc-3-nodes.yaml"
+        elif cluster_type == "mirror-3-dc-9-nodes":
+            template_path = "/ydbd_slice/baremetal/templates/mirror-3-dc-9-nodes.yaml"
+        else:
+            raise "Unreachable code"  # TODO(shmel1k@): improve error
+
+        f = rs.find(template_path).decode()
+        if args.output_file is not None and args.output_file != "":
+            with open(args.output_file, "w+") as f1:
+                f1.write(f)
+        else:
+            print(f)
+
+    mode = modes.add_parser(
+        "sample-config",
+        parents=[cluster_type_args(), output_file()],
+        description="Generate default mock-configuration for provided cluster-type"
     )
+
     mode.set_defaults(handler=_run)
 
 
@@ -1205,6 +1280,8 @@ def main(walle_provider=None):
         add_clear_mode(modes, walle_provider)
         add_format_mode(modes, walle_provider)
         add_explain_mode(modes, walle_provider)
+        add_sample_config_mode(modes)
+
         add_docker_build_mode(modes)
         add_kube_generate_mode(modes)
         add_kube_install_mode(modes)