LTXVid text2vid pipeline #208
Conversation
| @@ -213,8 +213,11 @@ def load_state_if_possible( | |||
| max_logging.log(f"restoring from this run's directory latest step {latest_step}") | |||
| try: | |||
| if not enable_single_replica_ckpt_restoring: | |||
| item = {checkpoint_item: orbax.checkpoint.args.PyTreeRestore(item=abstract_unboxed_pre_state)} | |||
| return checkpoint_manager.restore(latest_step, args=orbax.checkpoint.args.Composite(**item)) | |||
| if checkpoint_item == " ": | |||
There was a problem hiding this comment.
should this be if checkpoint_item is None?
There was a problem hiding this comment.
if checkpoint set to None, cannot pass the check "if checkpoint_manager and checkpoint_item:" in max_utils.py. So I set it to empty string to get around this
| pipeline = LTXVideoPipeline.from_pretrained(config, enhance_prompt=enhance_prompt) | ||
| if config.pipeline_type == "multi-scale": | ||
| pipeline = LTXMultiScalePipeline(pipeline) | ||
| # s0 = time.perf_counter() |
There was a problem hiding this comment.
remove commented out lines.
There was a problem hiding this comment.
Do we want to keep the time benchmarking in the code?
src/maxdiffusion/max_utils.py
Outdated
| @@ -402,7 +402,10 @@ def setup_initial_state( | |||
| config.enable_single_replica_ckpt_restoring, | |||
| ) | |||
| if state: | |||
| state = state[checkpoint_item] | |||
| if checkpoint_item == " ": | |||
There was a problem hiding this comment.
why is checkpoint_item checking against " " instead of None?
| skip_block_list=config.first_pass["skip_block_list"], | ||
| ) | ||
| latents = result | ||
| print("first pass done") |
There was a problem hiding this comment.
use max_logger for print statements
when I run these instructions, I get an error: Traceback (most recent call last):
File "/mnt/disks/external_disk/maxdiffusion/src/maxdiffusion/generate_ltx_video.py", line 20, in <module>
from maxdiffusion.pipelines.ltx_video.ltx_video_pipeline import LTXVideoPipeline
File "/mnt/disks/external_disk/maxdiffusion/src/maxdiffusion/pipelines/ltx_video/ltx_video_pipeline.py", line 20, in <module>
from maxdiffusion.models.ltx_video.autoencoders.vae_torchax import TorchaxCausalVideoAutoencoder
File "/mnt/disks/external_disk/maxdiffusion/src/maxdiffusion/models/ltx_video/autoencoders/vae_torchax.py", line 17, in <module>
from maxdiffusion.models.ltx_video.autoencoders.causal_video_autoencoder import CausalVideoAutoencoder
File "/mnt/disks/external_disk/maxdiffusion/src/maxdiffusion/models/ltx_video/autoencoders/causal_video_autoencoder.py", line 28, in <module>
from diffusers.utils import logging
ModuleNotFoundError: No module named 'diffusers' |
|
Do you need to update requirements.txt for new dependencies? |
Will need to install diffusers. Is this okay? I can add this in the requirements.txt |
Yes just update the requirements accordingly. |
…puter/maxdiffusion into vae-pipeline-cleaned
Running Instructions:
Create a new virtual environment with conda create, and run bash setup.sh MODE=stable DEVICE=tpu
python convert_torch_weights_to_jax.py --ckpt_path [LOCAL DIRECTORY FOR WEIGHTS] --transformer_config_path ../xora_v1.2-13B-balanced-128.json
python src/maxdiffusion/generate_ltx_video.py src/maxdiffusion/configs/ltx_video.yml output_dir="[SAME DIRECTORY]" config_path="src/maxdiffusion/models/ltx_video/xora_v1.2-13B-balanced-128.json"
Note: this should be with the quotes!
Other generation parameters can be set in ltx_video.yml file.