Merge pull request #27 from neph1/update-v0.14.0

Update v0.14.0

Author: neph1

config/config_categories.yaml

@@ -1,7 +1,8 @@
 Dataset: data_root, video_column, caption_column, dataset_file, id_token, image_resolution_buckets, video_resolution_buckets, caption_dropout_p, enable_precomputations, precomputation_once, precomputation_items, precomputation_dir
-Training: training_type, seed, train_steps, rank, lora_alpha, target_modules, gradient_accumulation_steps, checkpointing_steps, checkpointing_limit, enable_slicing, enable_tiling, batch_size, resume_from_checkpoint
+Training: training_type, seed, train_steps, gradient_accumulation_steps, checkpointing_steps, checkpointing_limit, enable_slicing, enable_tiling, batch_size, resume_from_checkpoint
 Optimizer: optimizer, lr, beta1, beta2, epsilon, weight_decay, max_grad_norm, lr_scheduler, lr_num_cycles, lr_warmup_steps
 Validation: validation_steps, num_validation_videos, validation_dataset_file
 Accelerate: gpu_ids, nccl_timeout, gradient_checkpointing, allow_tf32, dataloader_num_workers, report_to, accelerate_config
-Model: model_name, pretrained_model_name_or_path, text_encoder_dtype, text_encoder_2_dtype, text_encoder_3_dtype, vae_dtype, layerwise_upcasting_modules, layerwise_upcasting_storage_dtype, layerwise_upcasting_granularity, enable_model_cpu_offload
-Parallelism: parallel_backend, pp_degree, dp_degree, dp_shards, cp_degree, tp_degree, num_gpus, master_address, master_port, nproc_per_node
+Model: model_name, pretrained_model_name_or_path, text_encoder_dtype, text_encoder_2_dtype, text_encoder_3_dtype, vae_dtype, layerwise_upcasting_modules, layerwise_upcasting_storage_dtype, layerwise_upcasting_granularity, enable_model_cpu_offload, text_encoder_id, text_encoder_2_id, text_encoder_3_id, tokenizer_id, tokenizer_2_id, tokenized_3_id, transformer_id, vae_id
+Parallelism: parallel_backend, pp_degree, dp_degree, dp_shards, cp_degree, tp_degree, num_gpus, master_address, master_port, nproc_per_node
+Control: control_type, rank, lora_alpha, target_modules

config/config_template.yaml

@@ -8,6 +8,7 @@ caption_dropout_technique: 'empty'
 checkpointing_limit: 102
 checkpointing_steps: 500
 cp_degree: 1
+control_type: ['none', 'canny', 'custom']
 dataloader_num_workers: 0
 dataset_config: ''
 diffusion_options: '--flow_weighting_scheme logit_normal'
@@ -17,6 +18,8 @@ enable_model_cpu_offload: false
 enable_precomputation: false
 enable_slicing: true
 enable_tiling: true
+frame_conditioning_type: ['index', 'prefix', 'random', 'first_and_last', 'full']
+frame_conditioning_index: '0'
 epsilon: 1e-8
 gpu_ids: '0'
 gradient_accumulation_steps: 4
@@ -54,13 +57,21 @@ target_modules: to_q to_k to_v to_out.0
 text_encoder_dtype: [bf16, fp16, fp32]
 text_encoder_2_dtype: [bf16, fp16, fp32]
 text_encoder_3_dtype: [bf16, fp16, fp32]
+text_encoder_id: ''
+text_encoder_2_id: ''
+text_encoder_3_id: ''
+tokenizer_id: ''
+tokenizer_2_id: ''
+tokenizer_3_id: ''
 tp_degree: 1
 tracker_name: finetrainers
 transformer_dtype: [bf16, fp16, fp32]
+transformer_id: ''
 train_steps: 3000
-training_type: ['lora', 'full-finetune']
+training_type: ['lora', 'control-lora', 'full-finetune']
 use_8bit_bnb: false
 vae_dtype: [bf16, fp16, fp32]
+vae_id: ''
 validation_dataset_file: ''
 validation_steps: 100
 weight_decay: 0.001

pyproject.toml

@@ -1,9 +1,9 @@
 [project]
 name = "finetrainers-ui"
-version = "0.10.0"
+version = "0.13.0"
 dependencies = [
     "gradio",
-    "torch>=2.4.1"
+    "torch>=2.5.1"
 ]
 description = "A gradio based ui for training video transformer models with finetrainers as backend"
 readme = "README.md"

run_trainer.py

@@ -33,6 +33,22 @@ def run(self, config: Config, finetrainers_path: str, log_file: str):
                      "--text_encoder_3_dtype", config.get('text_encoder_3_dtype'),
                      "--transformer_dtype", config.get('transformer_dtype'),
                      "--vae_dtype", config.get('vae_dtype')]
+        if config.get('text_encoder_id'):
+            model_cmd += ["--text_encoder_id", config.get('text_encoder_id')]
+        if config.get('text_encoder_2_id'):
+            model_cmd += ["--text_encoder_2_id", config.get('text_encoder_2_id')]
+        if config.get('text_encoder_3_id'):
+            model_cmd += ["--text_encoder_3_id", config.get('text_encoder_3_id')]
+        if config.get('transformer_id'):
+            model_cmd += ["--transformer_id", config.get('transformer_id')]
+        if config.get('vae_id'):
+            model_cmd += ["--vae_id", config.get('vae_id')]
+        if config.get('tokenizer_id'):
+            model_cmd += ["--tokenizer_id", config.get('tokenizer_id')]
+        if config.get('tokenizer_2_id'):
+            model_cmd += ["--tokenizer_2_id", config.get('tokenizer_2_id')]
+        if config.get('tokenizer_3_id'):
+            model_cmd += ["--tokenizer_3_id", config.get('tokenizer_3_id')]
 
         if config.get('layerwise_upcasting_modules') != 'none':
             model_cmd +=["--layerwise_upcasting_modules", config.get('layerwise_upcasting_modules'),
@@ -55,10 +71,7 @@ def run(self, config: Config, finetrainers_path: str, log_file: str):
         training_cmd = ["--training_type", config.get('training_type'),
                 "--seed", config.get('seed'),
                 "--batch_size", config.get('batch_size'),
-                "--train_steps", config.get('train_steps'),
-                "--rank", config.get('rank'),
-                "--lora_alpha", config.get('lora_alpha'),
-                "--target_modules"]
+                "--train_steps", config.get('train_steps')]
         training_cmd += config.get('target_modules').split(' ')
         training_cmd += ["--gradient_accumulation_steps", config.get('gradient_accumulation_steps'),
                 '--gradient_checkpointing' if config.get('gradient_checkpointing') else '',
@@ -87,6 +100,12 @@ def run(self, config: Config, finetrainers_path: str, log_file: str):
         validation_cmd = ["--validation_dataset_file" if config.get('validation_dataset_file') else '',
                   "--num_validation_videos", config.get('num_validation_videos'),
                   "--validation_steps", config.get('validation_steps')]
+        
+        control_cmd = ["--rank", config.get('rank'),
+                "--lora_alpha", config.get('lora_alpha'),
+                "--control_type", config.get('control_type'),
+                "--frame_conditioning_index", config.get('frame_conditioning_index'),
+                "--frame_conditioning_type", config.get('frame_conditioning_type')]
 
         miscellaneous_cmd = ["--tracker_name", config.get('tracker_name'),
                      "--output_dir", config.get('output_dir'),
@@ -105,7 +124,7 @@ def run(self, config: Config, finetrainers_path: str, log_file: str):
             pre_command = ["accelerate", "launch", "--config_file", f"{finetrainers_path}/accelerate_configs/{config.get('accelerate_config')}", "--gpu_ids", config.get('gpu_ids')]
         elif parallel_backend == 'ptd':
             pre_command = ["torchrun", "--standalone", "--nnodes", num_gpus, "--nproc_per_node", config.get('nproc_per_node'), "--rdzv_backend", "c10d", "--rdzv_endpoint", f"{address}:{port}"]
-        cmd = pre_command + [f"{finetrainers_path}/train.py"] + parallel_cmd + model_cmd + dataset_cmd + dataloader_cmd + training_cmd + optimizer_cmd + validation_cmd + miscellaneous_cmd
+        cmd = pre_command + [f"{finetrainers_path}/train.py"] + parallel_cmd + model_cmd + dataset_cmd + dataloader_cmd + training_cmd + optimizer_cmd + validation_cmd + miscellaneous_cmd + control_cmd
         fixed_cmd = []
         for i in range(len(cmd)):
             if cmd[i] != '':