[feat][mdsv2] Support toml config.

Author: rock-git

conf/mdsv2.gflags renamed to conf/dingo-mdsv2.gflags

@@ -0,0 +1,25 @@
+[server]
+id = $INSTANCE_ID$
+addr = "$SERVER_HOST$:$SERVER_PORT$"
+
+[server.service.meta]
+read_worker_num = 128
+read_worker_max_pending_num = 1024
+read_worker_use_pthread = false
+
+write_worker_num = 128
+write_worker_max_pending_num = 1024
+write_worker_use_pthread = false
+
+
+[log]
+level = "INFO"
+path = "$BASE_PATH$/log"
+
+
+[crontab]
+heartbeat_interval_s = 5
+fsinfosync_interval_s = 10
+mdsmonitor_interval_s = 5
+quota_sync_interval_s = 6
+gc_interval_s = 60

conf/dingo-mdsv2.template.toml

@@ -66,18 +66,18 @@ function deploy_server() {
 
   if [ "${FLAGS_replace_conf}" == "0" ]; then
     # conf file
-    dist_conf="${dstpath}/conf/${role}.conf"
-    cp $srcpath/conf/${role}.template.conf $dist_conf
+    dist_conf="${dstpath}/conf/${server_name}.toml"
+    cp $srcpath/conf/${server_name}.template.toml $dist_conf
 
     sed  -i 's,\$INSTANCE_ID\$,'"$instance_id"',g'                  $dist_conf
     sed  -i 's,\$SERVER_HOST\$,'"$SERVER_HOST"',g'                  $dist_conf
     sed  -i 's,\$SERVER_PORT\$,'"$server_port"',g'                  $dist_conf
     sed  -i 's,\$BASE_PATH\$,'"$dstpath"',g'                        $dist_conf
 
     # gflags file
-    if [ -f $srcpath/conf/${role}.gflags ]
+    if [ -f $srcpath/conf/${server_name}.gflags ]
     then
-        cp $srcpath/conf/${role}.gflags $dstpath/conf/gflags
+        cp $srcpath/conf/${server_name}.gflags $dstpath/conf/
     fi
 
     # coor_list file

conf/mdsv2.conf

@@ -83,7 +83,7 @@ function start_server() {
   server_name="dingo-${role}"
   echo "start server: ${root_dir}/bin/${server_name}"
 
-  nohup ${root_dir}/bin/${server_name} 2>&1 >./log/out &
+  nohup ${root_dir}/bin/${server_name} --conf=${root_dir}/conf/${server_name}.toml 2>&1 >./log/out &
 }
 
 

conf/mdsv2.template.conf

@@ -71,7 +71,8 @@ def threaded_create_structure(base_path, depth, num_threads=5, num_dirs=10, num_
     with ThreadPoolExecutor(max_workers=num_threads) as executor:
         futures = []
         for i in range(num_threads):
-            thread_base_path = f"{base_path}_thread_{i}"
+            thread_base_path = f"{base_path}/thread_{i}"
+            create_directory(thread_base_path)  # 确保线程的基础路径存在
             futures.append(executor.submit(create_nested_structure, thread_base_path, depth, num_dirs, num_files))
 
         for future in futures:
@@ -83,7 +84,7 @@ def main():
     # 解析命令行参数
     parser = argparse.ArgumentParser(description="Test script for creating and deleting nested directory structures.")
     parser.add_argument('--path', type=str, default='test_structure', help='Base path for the nested structure')
-    parser.add_argument('--depth', type=int, default=3, help='Depth of the nested structure')
+    parser.add_argument('--depth', type=int, default=5, help='Depth of the nested structure')
     parser.add_argument('--threads', type=int, default=5, help='Number of threads to use for creating structure')
     args = parser.parse_args()
 
@@ -92,9 +93,11 @@ def main():
         # 运行各种测试
         base_path = args.path
         depth = args.depth
+        num_threads = args.threads
         print(f"Creating nested structure at {base_path} with depth {depth}...")
 
-        create_nested_structure(base_path, depth, num_dirs=5, num_files=1000)
+        # create_nested_structure(base_path, depth, num_dirs=5, num_files=1000)
+        threaded_create_structure(base_path, depth, num_threads, num_dirs=5, num_files=1000)
 
         print("\ntest finish!")
 

dev-scripts/deploy.sh

@@ -40,6 +40,7 @@ target_link_libraries(mdsv2_lib
   mdsv2_background
   mdsv2_statistics
 
+  dingofs_options
   dingofs_utils
 
   absl::log_internal_message

dev-scripts/start.sh

@@ -31,7 +31,7 @@ namespace mdsv2 {
 
 struct CrontabConfig {
   std::string name;
-  int32_t interval;
+  uint32_t interval;
   bool async;
   std::function<void(void*)> funcer;
 };

dev-scripts/test.py

@@ -259,6 +259,11 @@ class SimpleWorkerSet : public WorkerSet {
     return std::make_shared<SimpleWorkerSet>(name, worker_num, max_pending_task_count, use_pthread, is_inplace_run);
   }
 
+  static WorkerSetUPtr NewUnique(std::string name, uint32_t worker_num, uint32_t max_pending_task_count,
+                                 bool use_pthread, bool is_inplace_run) {
+    return std::make_unique<SimpleWorkerSet>(name, worker_num, max_pending_task_count, use_pthread, is_inplace_run);
+  }
+
   bool Init() override;
   void Destroy() override;