From b22708159617e910f3c00e5dc4f064b94ed09ef6 Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Wed, 12 Aug 2026 01:12:09 -0700 Subject: [PATCH 1/6] LTX-2.5 modular pipeline init docs --- docs/source/en/api/pipelines/ltx2.md | 147 +++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index 9b4c0bdf81e8..82d31aaf62ab 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -915,6 +915,133 @@ print(f"predicted {seconds:.2f}s -> {num_frames} frames") Converting a 2.5 checkpoint picks the head up automatically with `--full_pipeline`, or on its own with `--duration_head`. Checkpoints predating 2.5 have no such weights, and conversion skips the component rather than failing. +### LTX-2.5 Modular + +LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. Below is a T2V modular example: + +```py +import torch +from transformers import AutoModelForImageTextToText, AutoProcessor +from diffusers import ModularPipeline, ComponentsManager +from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor +from diffusers.pipelines.ltx2.utils import DEFAULT_NEGATIVE_PROMPT +from diffusers.utils import encode_video + +device = "cuda" +frame_rate = 24.0 +random_seed = 42 +generator = torch.Generator(device).manual_seed(random_seed) + +model_path = "Lightricks/LTX-2.5-Diffusers" +enhancer_model_id = "google/gemma-4-E2B-it" + +cm = ComponentsManager() +pipe = ModularPipeline.from_pretrained(model_path, components_manager=cm) +pipe.load_components(dtype=torch.bfloat16) +if getattr(pipe, "prompt_enhancer", None) is None: + prompt_enhancer = AutoModelForImageTextToText.from_pretrained(enhancer_model_id, dtype=torch.bfloat16) + processor = AutoProcessor.from_pretrained(enhancer_model_id) + pipe.update_components(prompt_enhancer=prompt_enhancer, processor=processor) +# Set memory_reserve_margin higher to more aggressively offload component models +cm.enable_auto_cpu_offload(device=device, memory_reserve_margin="20GB") +# The NATTEN processor works if `kernels` is available (`pip install kernels`) +# Otherwise omit the below line to use the Flex Attention processor +pipe.diffusion_decoder.set_attn_processor(LTX2VideoVaeNeighborhoodNattenProcessor()) +pipe.diffusion_decoder.enable_tiling() + +prompt = ( + "A cinematic shot of a red fox walking through a snowy forest at dawn, golden light filtering through pine trees." +) + +output_state = pipe( + prompt=prompt, + negative_prompt=DEFAULT_NEGATIVE_PROMPT, + width=768, + height=512, + num_frames=None, # Set to an int (e.g. 121) to specify a fixed video length + frame_rate=frame_rate, + num_inference_steps=30, + use_cross_timestep=True, + enable_prompt_enhancement=True, + generator=generator, + output_type="np", +) +video = output_state.get("videos") +audio = output_state.get("audio") + +encode_video( + video[0], + fps=frame_rate, + audio=audio[0].float().cpu(), + audio_sample_rate=pipe.vocoder.config.output_sampling_rate, + output_path="ltx2_5_modular_t2v.mp4", +) +``` + +The modular pipeline will automatically switch workflows based on the supplied inputs. For example, if `images` is supplied, an I2V workflow will be used: + +```py +import torch +from transformers import AutoModelForImageTextToText, AutoProcessor +from diffusers import ModularPipeline, ComponentsManager +from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor +from diffusers.pipelines.ltx2.utils import DEFAULT_NEGATIVE_PROMPT +from diffusers.utils import encode_video, load_image + +device = "cuda" +frame_rate = 24.0 +random_seed = 42 +generator = torch.Generator(device).manual_seed(random_seed) + +model_path = "Lightricks/LTX-2.5-Diffusers" +enhancer_model_id = "google/gemma-4-E2B-it" + +cm = ComponentsManager() +pipe = ModularPipeline.from_pretrained(model_path, components_manager=cm) +pipe.load_components(dtype=torch.bfloat16) +if getattr(pipe, "prompt_enhancer", None) is None: + prompt_enhancer = AutoModelForImageTextToText.from_pretrained(enhancer_model_id, dtype=torch.bfloat16) + processor = AutoProcessor.from_pretrained(enhancer_model_id) + pipe.update_components(prompt_enhancer=prompt_enhancer, processor=processor) +cm.enable_auto_cpu_offload(device=device, memory_reserve_margin="20GB") +pipe.diffusion_decoder.set_attn_processor(LTX2VideoVaeNeighborhoodNattenProcessor()) +pipe.diffusion_decoder.enable_tiling() + +prompt = ( + "An astronaut hatches from a fragile egg on the surface of the Moon, the shell cracking and peeling apart in " + "gentle low-gravity motion." +) +image_path = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg" +image = load_image(image_path) + +output_state = pipe( + image=image, + prompt=prompt, + negative_prompt=DEFAULT_NEGATIVE_PROMPT, + width=768, + height=512, + num_frames=None, # Set to an int (e.g. 121) to specify a fixed video length + frame_rate=frame_rate, + num_inference_steps=30, + use_cross_timestep=True, + enable_prompt_enhancement=True, + generator=generator, + output_type="np", +) +video = output_state.get("videos") +audio = output_state.get("audio") + +encode_video( + video[0], + fps=frame_rate, + audio=audio[0].float().cpu(), + audio_sample_rate=pipe.vocoder.config.output_sampling_rate, + output_path="ltx2_5_modular_i2v.mp4", +) +``` + +You can see the supported workflows in the docs for each blockset (e.g. `LTX2AutoBlocks`, `LTX25AutoBlocks`). + ## LTX2Pipeline [[autodoc]] LTX2Pipeline @@ -954,3 +1081,23 @@ Converting a 2.5 checkpoint picks the head up automatically with `--full_pipelin ## LTX2PipelineOutput [[autodoc]] pipelines.ltx2.pipeline_output.LTX2PipelineOutput + +## LTX2ModularPipeline + +[[autodoc]] LTX2ModularPipeline + +## LTX2AutoBlocks + +[[autodoc]] LTX2AutoBlocks + +## LTX25ModularPipeline + +[[autodoc]] LTX25ModularPipeline + +## LTX25AutoBlocks + +[[autodoc]] LTX25AutoBlocks + +## LTX2Guidance + +[[autodoc]] modular_pipelines.ltx2.guider.LTX2Guidance From 657bfcab0ba9be4ba78827ab3611bba0dc428f8f Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Thu, 20 Aug 2026 18:11:19 -0700 Subject: [PATCH 2/6] Update LTX-2.5 modular docs now that the modular pipeline automatic downloads the Gemma 4 prompt enhancer and processor --- docs/source/en/api/pipelines/ltx2.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index 82d31aaf62ab..ae8f292e7a8f 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -917,7 +917,7 @@ Converting a 2.5 checkpoint picks the head up automatically with `--full_pipelin ### LTX-2.5 Modular -LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. Below is a T2V modular example: +LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio [`LTX2Guidance`] `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will donwload the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: ```py import torch @@ -933,15 +933,10 @@ random_seed = 42 generator = torch.Generator(device).manual_seed(random_seed) model_path = "Lightricks/LTX-2.5-Diffusers" -enhancer_model_id = "google/gemma-4-E2B-it" cm = ComponentsManager() pipe = ModularPipeline.from_pretrained(model_path, components_manager=cm) pipe.load_components(dtype=torch.bfloat16) -if getattr(pipe, "prompt_enhancer", None) is None: - prompt_enhancer = AutoModelForImageTextToText.from_pretrained(enhancer_model_id, dtype=torch.bfloat16) - processor = AutoProcessor.from_pretrained(enhancer_model_id) - pipe.update_components(prompt_enhancer=prompt_enhancer, processor=processor) # Set memory_reserve_margin higher to more aggressively offload component models cm.enable_auto_cpu_offload(device=device, memory_reserve_margin="20GB") # The NATTEN processor works if `kernels` is available (`pip install kernels`) @@ -994,15 +989,10 @@ random_seed = 42 generator = torch.Generator(device).manual_seed(random_seed) model_path = "Lightricks/LTX-2.5-Diffusers" -enhancer_model_id = "google/gemma-4-E2B-it" cm = ComponentsManager() pipe = ModularPipeline.from_pretrained(model_path, components_manager=cm) pipe.load_components(dtype=torch.bfloat16) -if getattr(pipe, "prompt_enhancer", None) is None: - prompt_enhancer = AutoModelForImageTextToText.from_pretrained(enhancer_model_id, dtype=torch.bfloat16) - processor = AutoProcessor.from_pretrained(enhancer_model_id) - pipe.update_components(prompt_enhancer=prompt_enhancer, processor=processor) cm.enable_auto_cpu_offload(device=device, memory_reserve_margin="20GB") pipe.diffusion_decoder.set_attn_processor(LTX2VideoVaeNeighborhoodNattenProcessor()) pipe.diffusion_decoder.enable_tiling() @@ -1040,7 +1030,7 @@ encode_video( ) ``` -You can see the supported workflows in the docs for each blockset (e.g. `LTX2AutoBlocks`, `LTX25AutoBlocks`). +You can see the supported workflows in the docs for each blockset (e.g. [`LTX2AutoBlocks`], [`LTX25AutoBlocks`]). ## LTX2Pipeline From 6a2a2b1bf4fd96bf8f652037cac220d187dab6fc Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Thu, 20 Aug 2026 18:24:00 -0700 Subject: [PATCH 3/6] fix some modular docs typos --- docs/source/en/api/pipelines/ltx2.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index ae8f292e7a8f..37f71eab15ec 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -917,7 +917,7 @@ Converting a 2.5 checkpoint picks the head up automatically with `--full_pipelin ### LTX-2.5 Modular -LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio [`LTX2Guidance`] `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will donwload the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: +LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will donwload the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: ```py import torch @@ -973,7 +973,7 @@ encode_video( ) ``` -The modular pipeline will automatically switch workflows based on the supplied inputs. For example, if `images` is supplied, an I2V workflow will be used: +The modular pipeline will automatically switch workflows based on the supplied inputs. For example, if `image` is supplied, an I2V workflow will be used: ```py import torch From cf8333a88e8bf43278538649127156c3ec5e3227 Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Thu, 20 Aug 2026 18:45:35 -0700 Subject: [PATCH 4/6] fix more typos --- docs/source/en/api/pipelines/ltx2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index 37f71eab15ec..3aeb5d13e87d 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -917,7 +917,7 @@ Converting a 2.5 checkpoint picks the head up automatically with `--full_pipelin ### LTX-2.5 Modular -LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will donwload the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: +LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will download the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: ```py import torch From ede25cf1b5a15615f0e555a167d87af4eb89a730 Mon Sep 17 00:00:00 2001 From: dg845 <58458699+dg845@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:30:37 -0700 Subject: [PATCH 5/6] Update docs/source/en/api/pipelines/ltx2.md Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/api/pipelines/ltx2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index 3aeb5d13e87d..cdad5b77e929 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -917,7 +917,7 @@ Converting a 2.5 checkpoint picks the head up automatically with `--full_pipelin ### LTX-2.5 Modular -LTX-2.5 is also available as a modular pipeline. The LTX-2.5 modular pipeline is configured to use the diffusion decoder and duration prediction by default. It implements multimodal guidance (CFG + STG + modality isolation) via video and audio `LTX2Guidance` `guider` components; video- and audio-specific guidance parameters can be specified on their respective guiders. By default, the modular pipeline will download the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: +LTX-2.5 is also available as a modular pipeline. The default blockset uses the diffusion decoder and predicts the video duration when `num_frames` is omitted. It applies guidance separately to video and audio through the `guider` and `audio_guider` components. See [`LTX2Guidance`] for the available guidance parameters. By default, the modular pipeline will download the prompt enhancer and processor from the [google/gemma-4-E2B-it](https://huggingface.co/google/gemma-4-E2B-it) repo. Below is a T2V modular example: ```py import torch From 087018140d0b78f5a3ff4ebc80cc0b36670416e8 Mon Sep 17 00:00:00 2001 From: Daniel Gu Date: Fri, 21 Aug 2026 16:40:45 -0700 Subject: [PATCH 6/6] Remove unused transformers imports from LTX-2.5 modular examples --- docs/source/en/api/pipelines/ltx2.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/en/api/pipelines/ltx2.md b/docs/source/en/api/pipelines/ltx2.md index cdad5b77e929..e73689b4af52 100644 --- a/docs/source/en/api/pipelines/ltx2.md +++ b/docs/source/en/api/pipelines/ltx2.md @@ -921,7 +921,6 @@ LTX-2.5 is also available as a modular pipeline. The default blockset uses the d ```py import torch -from transformers import AutoModelForImageTextToText, AutoProcessor from diffusers import ModularPipeline, ComponentsManager from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor from diffusers.pipelines.ltx2.utils import DEFAULT_NEGATIVE_PROMPT @@ -977,7 +976,6 @@ The modular pipeline will automatically switch workflows based on the supplied i ```py import torch -from transformers import AutoModelForImageTextToText, AutoProcessor from diffusers import ModularPipeline, ComponentsManager from diffusers.models.autoencoders.ltx2_diffusion_decoder import LTX2VideoVaeNeighborhoodNattenProcessor from diffusers.pipelines.ltx2.utils import DEFAULT_NEGATIVE_PROMPT