diff --git a/tests/lora/test_lora_layers_cogvideox.py b/tests/lora/test_lora_layers_cogvideox.py deleted file mode 100644 index bf2a17f0c638..000000000000 --- a/tests/lora/test_lora_layers_cogvideox.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import unittest - -import torch -from parameterized import parameterized -from transformers import AutoTokenizer, T5EncoderModel - -from diffusers import ( - AutoencoderKLCogVideoX, - CogVideoXDPMScheduler, - CogVideoXPipeline, - CogVideoXTransformer3DModel, -) - -from ..testing_utils import ( - floats_tensor, - require_peft_backend, - require_torch_accelerator, -) - - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests # noqa: E402 - - -@require_peft_backend -class CogVideoXLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): - pipeline_class = CogVideoXPipeline - scheduler_cls = CogVideoXDPMScheduler - scheduler_kwargs = {"timestep_spacing": "trailing"} - - transformer_kwargs = { - "num_attention_heads": 4, - "attention_head_dim": 8, - "in_channels": 4, - "out_channels": 4, - "time_embed_dim": 2, - "text_embed_dim": 32, - "num_layers": 1, - "sample_width": 16, - "sample_height": 16, - "sample_frames": 9, - "patch_size": 2, - "temporal_compression_ratio": 4, - "max_text_seq_length": 16, - } - transformer_cls = CogVideoXTransformer3DModel - vae_kwargs = { - "in_channels": 3, - "out_channels": 3, - "down_block_types": ( - "CogVideoXDownBlock3D", - "CogVideoXDownBlock3D", - "CogVideoXDownBlock3D", - "CogVideoXDownBlock3D", - ), - "up_block_types": ( - "CogVideoXUpBlock3D", - "CogVideoXUpBlock3D", - "CogVideoXUpBlock3D", - "CogVideoXUpBlock3D", - ), - "block_out_channels": (8, 8, 8, 8), - "latent_channels": 4, - "layers_per_block": 1, - "norm_num_groups": 2, - "temporal_compression_ratio": 4, - } - vae_cls = AutoencoderKLCogVideoX - tokenizer_cls, tokenizer_id = AutoTokenizer, "hf-internal-testing/tiny-random-t5" - text_encoder_cls, text_encoder_id = T5EncoderModel, "hf-internal-testing/tiny-random-t5" - - text_encoder_target_modules = ["q", "k", "v", "o"] - - supports_text_encoder_loras = False - - @property - def output_shape(self): - return (1, 9, 16, 16, 3) - - def get_dummy_inputs(self, with_generator=True): - batch_size = 1 - sequence_length = 16 - num_channels = 4 - num_frames = 9 - num_latent_frames = 3 # (num_frames - 1) // temporal_compression_ratio + 1 - sizes = (2, 2) - - generator = torch.manual_seed(0) - noise = floats_tensor((batch_size, num_latent_frames, num_channels) + sizes) - input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) - - pipeline_inputs = { - "prompt": "dance monkey", - "num_frames": num_frames, - "num_inference_steps": 4, - "guidance_scale": 6.0, - # Cannot reduce because convolution kernel becomes bigger than sample - "height": 16, - "width": 16, - "max_sequence_length": sequence_length, - "output_type": "np", - } - if with_generator: - pipeline_inputs.update({"generator": generator}) - - return noise, input_ids, pipeline_inputs - - def test_simple_inference_with_text_lora_denoiser_fused_multi(self): - super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3) - - def test_simple_inference_with_text_denoiser_lora_unfused(self): - super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3) - - def test_lora_scale_kwargs_match_fusion(self): - super().test_lora_scale_kwargs_match_fusion(expected_atol=9e-3, expected_rtol=9e-3) - - @parameterized.expand([("block_level", True), ("leaf_level", False)]) - @require_torch_accelerator - def test_group_offloading_inference_denoiser(self, offload_type, use_stream): - # TODO: We don't run the (leaf_level, True) test here that is enabled for other models. - # The reason for this can be found here: https://github.com/huggingface/diffusers/pull/11804#issuecomment-3013325338 - super()._test_group_offloading_inference_denoiser(offload_type, use_stream) - - @unittest.skip("Not supported in CogVideoX.") - def test_simple_inference_with_text_denoiser_block_scale(self): - pass - - @unittest.skip("Not supported in CogVideoX.") - def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self): - pass - - @unittest.skip("Not supported in CogVideoX.") - def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self): - pass diff --git a/tests/lora/test_lora_layers_flux2.py b/tests/lora/test_lora_layers_flux2.py deleted file mode 100644 index 33500709a6c7..000000000000 --- a/tests/lora/test_lora_layers_flux2.py +++ /dev/null @@ -1,105 +0,0 @@ -# coding=utf-8 -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import unittest - -import torch -from transformers import AutoProcessor, Mistral3ForConditionalGeneration - -from diffusers import AutoencoderKLFlux2, FlowMatchEulerDiscreteScheduler, Flux2Pipeline, Flux2Transformer2DModel - -from ..testing_utils import floats_tensor, require_peft_backend - - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests # noqa: E402 - - -@require_peft_backend -class Flux2LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): - pipeline_class = Flux2Pipeline - scheduler_cls = FlowMatchEulerDiscreteScheduler - scheduler_kwargs = {} - - transformer_kwargs = { - "patch_size": 1, - "in_channels": 4, - "num_layers": 1, - "num_single_layers": 1, - "attention_head_dim": 16, - "num_attention_heads": 2, - "joint_attention_dim": 16, - "timestep_guidance_channels": 256, - "axes_dims_rope": [4, 4, 4, 4], - } - transformer_cls = Flux2Transformer2DModel - vae_kwargs = { - "sample_size": 32, - "in_channels": 3, - "out_channels": 3, - "down_block_types": ("DownEncoderBlock2D",), - "up_block_types": ("UpDecoderBlock2D",), - "block_out_channels": (4,), - "layers_per_block": 1, - "latent_channels": 1, - "norm_num_groups": 1, - "use_quant_conv": False, - "use_post_quant_conv": False, - } - vae_cls = AutoencoderKLFlux2 - - tokenizer_cls, tokenizer_id = AutoProcessor, "hf-internal-testing/tiny-mistral3-diffusers" - text_encoder_cls, text_encoder_id = Mistral3ForConditionalGeneration, "hf-internal-testing/tiny-mistral3-diffusers" - denoiser_target_modules = ["to_qkv_mlp_proj", "to_k"] - - supports_text_encoder_loras = False - - @property - def output_shape(self): - return (1, 8, 8, 3) - - def get_dummy_inputs(self, with_generator=True): - batch_size = 1 - sequence_length = 10 - num_channels = 4 - sizes = (32, 32) - - generator = torch.manual_seed(0) - noise = floats_tensor((batch_size, num_channels) + sizes) - input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) - - pipeline_inputs = { - "prompt": "a dog is dancing", - "num_inference_steps": 2, - "guidance_scale": 5.0, - "height": 8, - "width": 8, - "max_sequence_length": 8, - "output_type": "np", - "text_encoder_out_layers": (1,), - } - if with_generator: - pipeline_inputs.update({"generator": generator}) - - return noise, input_ids, pipeline_inputs - - @unittest.skip("Not supported in Flux2.") - def test_simple_inference_with_text_denoiser_block_scale(self): - pass - - @unittest.skip("Not supported in Flux2.") - def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self): - pass diff --git a/tests/lora/test_lora_layers_qwenimage.py b/tests/lora/test_lora_layers_qwenimage.py deleted file mode 100644 index 7910774dbae0..000000000000 --- a/tests/lora/test_lora_layers_qwenimage.py +++ /dev/null @@ -1,107 +0,0 @@ -# coding=utf-8 -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import unittest - -import torch -from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer - -from diffusers import ( - AutoencoderKLQwenImage, - FlowMatchEulerDiscreteScheduler, - QwenImagePipeline, - QwenImageTransformer2DModel, -) - -from ..testing_utils import floats_tensor, require_peft_backend - - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests # noqa: E402 - - -@require_peft_backend -class QwenImageLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): - pipeline_class = QwenImagePipeline - scheduler_cls = FlowMatchEulerDiscreteScheduler - scheduler_kwargs = {} - - transformer_kwargs = { - "patch_size": 2, - "in_channels": 16, - "out_channels": 4, - "num_layers": 2, - "attention_head_dim": 16, - "num_attention_heads": 3, - "joint_attention_dim": 16, - "guidance_embeds": False, - "axes_dims_rope": (8, 4, 4), - } - transformer_cls = QwenImageTransformer2DModel - z_dim = 4 - vae_kwargs = { - "base_dim": z_dim * 6, - "z_dim": z_dim, - "dim_mult": [1, 2, 4], - "num_res_blocks": 1, - "temperal_downsample": [False, True], - "latents_mean": [0.0] * 4, - "latents_std": [1.0] * 4, - } - vae_cls = AutoencoderKLQwenImage - tokenizer_cls, tokenizer_id = Qwen2Tokenizer, "hf-internal-testing/tiny-random-Qwen25VLForCondGen" - text_encoder_cls, text_encoder_id = ( - Qwen2_5_VLForConditionalGeneration, - "hf-internal-testing/tiny-random-Qwen25VLForCondGen", - ) - denoiser_target_modules = ["to_q", "to_k", "to_v", "to_out.0"] - - supports_text_encoder_loras = False - - @property - def output_shape(self): - return (1, 8, 8, 3) - - def get_dummy_inputs(self, with_generator=True): - batch_size = 1 - sequence_length = 10 - num_channels = 4 - sizes = (32, 32) - - generator = torch.manual_seed(0) - noise = floats_tensor((batch_size, num_channels) + sizes) - input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) - - pipeline_inputs = { - "prompt": "A painting of a squirrel eating a burger", - "num_inference_steps": 4, - "guidance_scale": 0.0, - "height": 8, - "width": 8, - "output_type": "np", - } - if with_generator: - pipeline_inputs.update({"generator": generator}) - - return noise, input_ids, pipeline_inputs - - @unittest.skip("Not supported in Qwen Image.") - def test_simple_inference_with_text_denoiser_block_scale(self): - pass - - @unittest.skip("Not supported in Qwen Image.") - def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self): - pass diff --git a/tests/lora/test_lora_layers_sd.py b/tests/lora/test_lora_layers_sd.py deleted file mode 100644 index 4662075cfd0a..000000000000 --- a/tests/lora/test_lora_layers_sd.py +++ /dev/null @@ -1,769 +0,0 @@ -# coding=utf-8 -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import gc -import sys -import unittest - -import numpy as np -import torch -import torch.nn as nn -from huggingface_hub import hf_hub_download -from safetensors.torch import load_file -from transformers import CLIPTextModel, CLIPTokenizer - -from diffusers import ( - AutoPipelineForImage2Image, - AutoPipelineForText2Image, - DDIMScheduler, - DiffusionPipeline, - LCMScheduler, - StableDiffusionPipeline, -) -from diffusers.utils.import_utils import is_accelerate_available - -from ..testing_utils import ( - Expectations, - backend_empty_cache, - load_image, - nightly, - numpy_cosine_similarity_distance, - require_peft_backend, - require_torch_accelerator, - slow, - torch_device, -) - - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests, check_if_lora_correctly_set # noqa: E402 - - -if is_accelerate_available(): - from accelerate.utils import release_memory - - -class StableDiffusionLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase): - pipeline_class = StableDiffusionPipeline - scheduler_cls = DDIMScheduler - scheduler_kwargs = { - "beta_start": 0.00085, - "beta_end": 0.012, - "beta_schedule": "scaled_linear", - "clip_sample": False, - "set_alpha_to_one": False, - "steps_offset": 1, - } - unet_kwargs = { - "block_out_channels": (32, 64), - "layers_per_block": 2, - "sample_size": 32, - "in_channels": 4, - "out_channels": 4, - "down_block_types": ("DownBlock2D", "CrossAttnDownBlock2D"), - "up_block_types": ("CrossAttnUpBlock2D", "UpBlock2D"), - "cross_attention_dim": 32, - } - vae_kwargs = { - "block_out_channels": [32, 64], - "in_channels": 3, - "out_channels": 3, - "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], - "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], - "latent_channels": 4, - } - text_encoder_cls, text_encoder_id = CLIPTextModel, "peft-internal-testing/tiny-clip-text-2" - tokenizer_cls, tokenizer_id = CLIPTokenizer, "peft-internal-testing/tiny-clip-text-2" - - @property - def output_shape(self): - return (1, 64, 64, 3) - - def setUp(self): - super().setUp() - gc.collect() - backend_empty_cache(torch_device) - - def tearDown(self): - super().tearDown() - gc.collect() - backend_empty_cache(torch_device) - - # Keeping this test here makes sense because it doesn't look any integration - # (value assertions on logits). - @slow - @require_torch_accelerator - def test_integration_move_lora_cpu(self): - path = "stable-diffusion-v1-5/stable-diffusion-v1-5" - lora_id = "takuma104/lora-test-text-encoder-lora-target" - - pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) - pipe.load_lora_weights(lora_id, adapter_name="adapter-1") - pipe.load_lora_weights(lora_id, adapter_name="adapter-2") - pipe = pipe.to(torch_device) - - self.assertTrue( - check_if_lora_correctly_set(pipe.text_encoder), - "Lora not correctly set in text encoder", - ) - - self.assertTrue( - check_if_lora_correctly_set(pipe.unet), - "Lora not correctly set in unet", - ) - - # We will offload the first adapter in CPU and check if the offloading - # has been performed correctly - pipe.set_lora_device(["adapter-1"], "cpu") - - for name, module in pipe.unet.named_modules(): - if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device == torch.device("cpu")) - elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device != torch.device("cpu")) - - for name, module in pipe.text_encoder.named_modules(): - if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device == torch.device("cpu")) - elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device != torch.device("cpu")) - - pipe.set_lora_device(["adapter-1"], 0) - - for n, m in pipe.unet.named_modules(): - if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): - self.assertTrue(m.weight.device != torch.device("cpu")) - - for n, m in pipe.text_encoder.named_modules(): - if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): - self.assertTrue(m.weight.device != torch.device("cpu")) - - pipe.set_lora_device(["adapter-1", "adapter-2"], torch_device) - - for n, m in pipe.unet.named_modules(): - if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): - self.assertTrue(m.weight.device != torch.device("cpu")) - - for n, m in pipe.text_encoder.named_modules(): - if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): - self.assertTrue(m.weight.device != torch.device("cpu")) - - @slow - @require_torch_accelerator - def test_integration_move_lora_dora_cpu(self): - from peft import LoraConfig - - path = "stable-diffusion-v1-5/stable-diffusion-v1-5" - unet_lora_config = LoraConfig( - init_lora_weights="gaussian", - target_modules=["to_k", "to_q", "to_v", "to_out.0"], - use_dora=True, - ) - text_lora_config = LoraConfig( - init_lora_weights="gaussian", - target_modules=["q_proj", "k_proj", "v_proj", "out_proj"], - use_dora=True, - ) - - pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) - pipe.unet.add_adapter(unet_lora_config, "adapter-1") - pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") - - self.assertTrue( - check_if_lora_correctly_set(pipe.text_encoder), - "Lora not correctly set in text encoder", - ) - - self.assertTrue( - check_if_lora_correctly_set(pipe.unet), - "Lora not correctly set in unet", - ) - - for name, param in pipe.unet.named_parameters(): - if "lora_" in name: - self.assertEqual(param.device, torch.device("cpu")) - - for name, param in pipe.text_encoder.named_parameters(): - if "lora_" in name: - self.assertEqual(param.device, torch.device("cpu")) - - pipe.set_lora_device(["adapter-1"], torch_device) - - for name, param in pipe.unet.named_parameters(): - if "lora_" in name: - self.assertNotEqual(param.device, torch.device("cpu")) - - for name, param in pipe.text_encoder.named_parameters(): - if "lora_" in name: - self.assertNotEqual(param.device, torch.device("cpu")) - - @slow - @require_torch_accelerator - def test_integration_set_lora_device_different_target_layers(self): - # fixes a bug that occurred when calling set_lora_device with multiple adapters loaded that target different - # layers, see #11833 - from peft import LoraConfig - - path = "stable-diffusion-v1-5/stable-diffusion-v1-5" - pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) - # configs partly target the same, partly different layers - config0 = LoraConfig(target_modules=["to_k", "to_v"]) - config1 = LoraConfig(target_modules=["to_k", "to_q"]) - pipe.unet.add_adapter(config0, adapter_name="adapter-0") - pipe.unet.add_adapter(config1, adapter_name="adapter-1") - pipe = pipe.to(torch_device) - - self.assertTrue( - check_if_lora_correctly_set(pipe.unet), - "Lora not correctly set in unet", - ) - - # sanity check that the adapters don't target the same layers, otherwise the test passes even without the fix - modules_adapter_0 = {n for n, _ in pipe.unet.named_modules() if n.endswith(".adapter-0")} - modules_adapter_1 = {n for n, _ in pipe.unet.named_modules() if n.endswith(".adapter-1")} - self.assertNotEqual(modules_adapter_0, modules_adapter_1) - self.assertTrue(modules_adapter_0 - modules_adapter_1) - self.assertTrue(modules_adapter_1 - modules_adapter_0) - - # setting both separately works - pipe.set_lora_device(["adapter-0"], "cpu") - pipe.set_lora_device(["adapter-1"], "cpu") - - for name, module in pipe.unet.named_modules(): - if "adapter-0" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device == torch.device("cpu")) - elif "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device == torch.device("cpu")) - - # setting both at once also works - pipe.set_lora_device(["adapter-0", "adapter-1"], torch_device) - - for name, module in pipe.unet.named_modules(): - if "adapter-0" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device != torch.device("cpu")) - elif "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): - self.assertTrue(module.weight.device != torch.device("cpu")) - - -@slow -@nightly -@require_torch_accelerator -@require_peft_backend -class LoraIntegrationTests(unittest.TestCase): - def setUp(self): - super().setUp() - gc.collect() - backend_empty_cache(torch_device) - - def tearDown(self): - super().tearDown() - gc.collect() - backend_empty_cache(torch_device) - - def test_integration_logits_with_scale(self): - path = "stable-diffusion-v1-5/stable-diffusion-v1-5" - lora_id = "takuma104/lora-test-text-encoder-lora-target" - - pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) - pipe.load_lora_weights(lora_id) - pipe = pipe.to(torch_device) - - self.assertTrue( - check_if_lora_correctly_set(pipe.text_encoder), - "Lora not correctly set in text encoder", - ) - - prompt = "a red sks dog" - - images = pipe( - prompt=prompt, - num_inference_steps=15, - cross_attention_kwargs={"scale": 0.5}, - generator=torch.manual_seed(0), - output_type="np", - ).images - - expected_slice_scale = np.array([0.307, 0.283, 0.310, 0.310, 0.300, 0.314, 0.336, 0.314, 0.321]) - predicted_slice = images[0, -3:, -3:, -1].flatten() - - max_diff = numpy_cosine_similarity_distance(expected_slice_scale, predicted_slice) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_integration_logits_no_scale(self): - path = "stable-diffusion-v1-5/stable-diffusion-v1-5" - lora_id = "takuma104/lora-test-text-encoder-lora-target" - - pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) - pipe.load_lora_weights(lora_id) - pipe = pipe.to(torch_device) - - self.assertTrue( - check_if_lora_correctly_set(pipe.text_encoder), - "Lora not correctly set in text encoder", - ) - - prompt = "a red sks dog" - - images = pipe(prompt=prompt, num_inference_steps=30, generator=torch.manual_seed(0), output_type="np").images - - expected_slice_scale = np.array([0.074, 0.064, 0.073, 0.0842, 0.069, 0.0641, 0.0794, 0.076, 0.084]) - predicted_slice = images[0, -3:, -3:, -1].flatten() - - max_diff = numpy_cosine_similarity_distance(expected_slice_scale, predicted_slice) - - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_dreambooth_old_format(self): - generator = torch.Generator("cpu").manual_seed(0) - - lora_model_id = "hf-internal-testing/lora_dreambooth_dog_example" - - base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" - - pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) - pipe = pipe.to(torch_device) - pipe.load_lora_weights(lora_model_id) - - images = pipe( - "A photo of a sks dog floating in the river", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.7207, 0.6787, 0.6010, 0.7478, 0.6838, 0.6064, 0.6984, 0.6443, 0.5785]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-4 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_dreambooth_text_encoder_new_format(self): - generator = torch.Generator().manual_seed(0) - - lora_model_id = "hf-internal-testing/lora-trained" - - base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" - - pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) - pipe = pipe.to(torch_device) - pipe.load_lora_weights(lora_model_id) - - images = pipe("A photo of a sks dog", output_type="np", generator=generator, num_inference_steps=2).images - - images = images[0, -3:, -3:, -1].flatten() - - expected = np.array([0.6628, 0.6138, 0.5390, 0.6625, 0.6130, 0.5463, 0.6166, 0.5788, 0.5359]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-4 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_a1111(self): - generator = torch.Generator().manual_seed(0) - - pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None).to( - torch_device - ) - lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" - lora_filename = "light_and_shadow.safetensors" - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - - images = pipe( - "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_lycoris(self): - generator = torch.Generator().manual_seed(0) - - pipe = StableDiffusionPipeline.from_pretrained( - "hf-internal-testing/Amixx", safety_checker=None, use_safetensors=True, variant="fp16" - ).to(torch_device) - lora_model_id = "hf-internal-testing/edgLycorisMugler-light" - lora_filename = "edgLycorisMugler-light.safetensors" - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - - images = pipe( - "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.6463, 0.658, 0.599, 0.6542, 0.6512, 0.6213, 0.658, 0.6485, 0.6017]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_a1111_with_model_cpu_offload(self): - generator = torch.Generator().manual_seed(0) - - pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) - pipe.enable_model_cpu_offload(device=torch_device) - lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" - lora_filename = "light_and_shadow.safetensors" - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - - images = pipe( - "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_a1111_with_sequential_cpu_offload(self): - generator = torch.Generator().manual_seed(0) - - pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) - pipe.enable_sequential_cpu_offload(device=torch_device) - lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" - lora_filename = "light_and_shadow.safetensors" - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - - images = pipe( - "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_kohya_sd_v15_with_higher_dimensions(self): - generator = torch.Generator().manual_seed(0) - - pipe = StableDiffusionPipeline.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None - ).to(torch_device) - lora_model_id = "hf-internal-testing/urushisato-lora" - lora_filename = "urushisato_v15.safetensors" - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - - images = pipe( - "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 - ).images - - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.7165, 0.6616, 0.5833, 0.7504, 0.6718, 0.587, 0.6871, 0.6361, 0.5694]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_vanilla_funetuning(self): - generator = torch.Generator().manual_seed(0) - - lora_model_id = "hf-internal-testing/sd-model-finetuned-lora-t4" - - base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" - - pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) - pipe = pipe.to(torch_device) - pipe.load_lora_weights(lora_model_id) - - images = pipe("A pokemon with blue eyes.", output_type="np", generator=generator, num_inference_steps=2).images - - image_slice = images[0, -3:, -3:, -1].flatten() - - expected_slices = Expectations( - { - ("xpu", 3): np.array( - [ - 0.6544, - 0.6127, - 0.5397, - 0.6845, - 0.6047, - 0.5469, - 0.6349, - 0.5906, - 0.5382, - ] - ), - ("cuda", 7): np.array( - [ - 0.7406, - 0.699, - 0.5963, - 0.7493, - 0.7045, - 0.6096, - 0.6886, - 0.6388, - 0.583, - ] - ), - ("cuda", 8): np.array( - [ - 0.6542, - 0.61253, - 0.5396, - 0.6843, - 0.6044, - 0.5468, - 0.6349, - 0.5905, - 0.5381, - ] - ), - } - ) - expected_slice = expected_slices.get_expectation() - - max_diff = numpy_cosine_similarity_distance(expected_slice, image_slice) - assert max_diff < 1e-4 - - pipe.unload_lora_weights() - release_memory(pipe) - - def test_unload_kohya_lora(self): - generator = torch.manual_seed(0) - prompt = "masterpiece, best quality, mountain" - num_inference_steps = 2 - - pipe = StableDiffusionPipeline.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None - ).to(torch_device) - initial_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - initial_images = initial_images[0, -3:, -3:, -1].flatten() - - lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" - lora_filename = "Colored_Icons_by_vizsumit.safetensors" - - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - generator = torch.manual_seed(0) - lora_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - lora_images = lora_images[0, -3:, -3:, -1].flatten() - - pipe.unload_lora_weights() - generator = torch.manual_seed(0) - unloaded_lora_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() - - self.assertFalse(np.allclose(initial_images, lora_images)) - self.assertTrue(np.allclose(initial_images, unloaded_lora_images, atol=1e-3)) - - release_memory(pipe) - - def test_load_unload_load_kohya_lora(self): - # This test ensures that a Kohya-style LoRA can be safely unloaded and then loaded - # without introducing any side-effects. Even though the test uses a Kohya-style - # LoRA, the underlying adapter handling mechanism is format-agnostic. - generator = torch.manual_seed(0) - prompt = "masterpiece, best quality, mountain" - num_inference_steps = 2 - - pipe = StableDiffusionPipeline.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None - ).to(torch_device) - initial_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - initial_images = initial_images[0, -3:, -3:, -1].flatten() - - lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" - lora_filename = "Colored_Icons_by_vizsumit.safetensors" - - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - generator = torch.manual_seed(0) - lora_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - lora_images = lora_images[0, -3:, -3:, -1].flatten() - - pipe.unload_lora_weights() - generator = torch.manual_seed(0) - unloaded_lora_images = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() - - self.assertFalse(np.allclose(initial_images, lora_images)) - self.assertTrue(np.allclose(initial_images, unloaded_lora_images, atol=1e-3)) - - # make sure we can load a LoRA again after unloading and they don't have - # any undesired effects. - pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) - generator = torch.manual_seed(0) - lora_images_again = pipe( - prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps - ).images - lora_images_again = lora_images_again[0, -3:, -3:, -1].flatten() - - self.assertTrue(np.allclose(lora_images, lora_images_again, atol=1e-3)) - release_memory(pipe) - - def test_not_empty_state_dict(self): - # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again - pipe = AutoPipelineForText2Image.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 - ).to(torch_device) - pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) - - cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") - lcm_lora = load_file(cached_file) - - pipe.load_lora_weights(lcm_lora, adapter_name="lcm") - self.assertTrue(lcm_lora != {}) - release_memory(pipe) - - def test_load_unload_load_state_dict(self): - # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again - pipe = AutoPipelineForText2Image.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 - ).to(torch_device) - pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) - - cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") - lcm_lora = load_file(cached_file) - previous_state_dict = lcm_lora.copy() - - pipe.load_lora_weights(lcm_lora, adapter_name="lcm") - self.assertDictEqual(lcm_lora, previous_state_dict) - - pipe.unload_lora_weights() - pipe.load_lora_weights(lcm_lora, adapter_name="lcm") - self.assertDictEqual(lcm_lora, previous_state_dict) - - release_memory(pipe) - - def test_sdv1_5_lcm_lora(self): - pipe = DiffusionPipeline.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 - ) - pipe.to(torch_device) - pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) - - generator = torch.Generator("cpu").manual_seed(0) - - lora_model_id = "latent-consistency/lcm-lora-sdv1-5" - pipe.load_lora_weights(lora_model_id) - - image = pipe( - "masterpiece, best quality, mountain", generator=generator, num_inference_steps=4, guidance_scale=0.5 - ).images[0] - - expected_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora.png" - ) - - image_np = pipe.image_processor.pil_to_numpy(image) - expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) - - max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) - assert max_diff < 1e-4 - - pipe.unload_lora_weights() - - release_memory(pipe) - - def test_sdv1_5_lcm_lora_img2img(self): - pipe = AutoPipelineForImage2Image.from_pretrained( - "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 - ) - pipe.to(torch_device) - pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) - - init_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape.png" - ) - - generator = torch.Generator("cpu").manual_seed(0) - - lora_model_id = "latent-consistency/lcm-lora-sdv1-5" - pipe.load_lora_weights(lora_model_id) - - image = pipe( - "snowy mountain", - generator=generator, - image=init_image, - strength=0.5, - num_inference_steps=4, - guidance_scale=0.5, - ).images[0] - - expected_image = load_image( - "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora_img2img.png" - ) - - image_np = pipe.image_processor.pil_to_numpy(image) - expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) - - max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) - assert max_diff < 1e-4 - - pipe.unload_lora_weights() - - release_memory(pipe) - - def test_sd_load_civitai_empty_network_alpha(self): - """ - This test simply checks that loading a LoRA with an empty network alpha works fine - See: https://github.com/huggingface/diffusers/issues/5606 - """ - pipeline = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") - pipeline.enable_sequential_cpu_offload(device=torch_device) - civitai_path = hf_hub_download("ybelkada/test-ahi-civitai", "ahi_lora_weights.safetensors") - pipeline.load_lora_weights(civitai_path, adapter_name="ahri") - - images = pipeline( - "ahri, masterpiece, league of legends", - output_type="np", - generator=torch.manual_seed(156), - num_inference_steps=5, - ).images - images = images[0, -3:, -3:, -1].flatten() - expected = np.array([0.0, 0.0, 0.0, 0.002557, 0.020954, 0.001792, 0.006581, 0.00591, 0.002995]) - - max_diff = numpy_cosine_similarity_distance(expected, images) - assert max_diff < 1e-3 - - pipeline.unload_lora_weights() - release_memory(pipeline) diff --git a/tests/lora/test_lora_layers_wan.py b/tests/lora/test_lora_layers_wan.py deleted file mode 100644 index c1077d509238..000000000000 --- a/tests/lora/test_lora_layers_wan.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import unittest - -import torch -from transformers import AutoTokenizer, T5EncoderModel - -from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanPipeline, WanTransformer3DModel - -from ..testing_utils import floats_tensor, require_peft_backend, skip_mps - - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests # noqa: E402 - - -@require_peft_backend -@skip_mps -class WanLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): - pipeline_class = WanPipeline - scheduler_cls = FlowMatchEulerDiscreteScheduler - scheduler_kwargs = {} - - transformer_kwargs = { - "patch_size": (1, 2, 2), - "num_attention_heads": 2, - "attention_head_dim": 12, - "in_channels": 16, - "out_channels": 16, - "text_dim": 32, - "freq_dim": 256, - "ffn_dim": 32, - "num_layers": 2, - "cross_attn_norm": True, - "qk_norm": "rms_norm_across_heads", - "rope_max_seq_len": 32, - } - transformer_cls = WanTransformer3DModel - vae_kwargs = { - "base_dim": 3, - "z_dim": 16, - "dim_mult": [1, 1, 1, 1], - "num_res_blocks": 1, - "temperal_downsample": [False, True, True], - } - vae_cls = AutoencoderKLWan - has_two_text_encoders = True - tokenizer_cls, tokenizer_id = AutoTokenizer, "hf-internal-testing/tiny-random-t5" - text_encoder_cls, text_encoder_id = T5EncoderModel, "hf-internal-testing/tiny-random-t5" - - text_encoder_target_modules = ["q", "k", "v", "o"] - - supports_text_encoder_loras = False - - @property - def output_shape(self): - return (1, 9, 32, 32, 3) - - def get_dummy_inputs(self, with_generator=True): - batch_size = 1 - sequence_length = 16 - num_channels = 4 - num_frames = 9 - num_latent_frames = 3 # (num_frames - 1) // temporal_compression_ratio + 1 - sizes = (4, 4) - - generator = torch.manual_seed(0) - noise = floats_tensor((batch_size, num_latent_frames, num_channels) + sizes) - input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) - - pipeline_inputs = { - "prompt": "", - "num_frames": num_frames, - "num_inference_steps": 1, - "guidance_scale": 6.0, - "height": 32, - "width": 32, - "max_sequence_length": sequence_length, - "output_type": "np", - } - if with_generator: - pipeline_inputs.update({"generator": generator}) - - return noise, input_ids, pipeline_inputs - - def test_simple_inference_with_text_lora_denoiser_fused_multi(self): - super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3) - - def test_simple_inference_with_text_denoiser_lora_unfused(self): - super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3) - - @unittest.skip("Not supported in Wan.") - def test_simple_inference_with_text_denoiser_block_scale(self): - pass - - @unittest.skip("Not supported in Wan.") - def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self): - pass diff --git a/tests/lora/test_lora_layers_wanvace.py b/tests/lora/test_lora_layers_wanvace.py deleted file mode 100644 index c17d0d07a0e3..000000000000 --- a/tests/lora/test_lora_layers_wanvace.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import tempfile -import unittest - -import numpy as np -import safetensors.torch -import torch -from PIL import Image -from transformers import AutoTokenizer, T5EncoderModel - -from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanVACEPipeline, WanVACETransformer3DModel -from diffusers.utils.import_utils import is_peft_available - -from ..testing_utils import ( - floats_tensor, - require_peft_backend, - require_peft_version_greater, - skip_mps, - torch_device, -) - - -if is_peft_available(): - from peft.utils import get_peft_model_state_dict - -sys.path.append(".") - -from .utils import PeftLoraLoaderMixinTests # noqa: E402 - - -@require_peft_backend -@skip_mps -class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): - pipeline_class = WanVACEPipeline - scheduler_cls = FlowMatchEulerDiscreteScheduler - scheduler_kwargs = {} - - transformer_kwargs = { - "patch_size": (1, 2, 2), - "num_attention_heads": 2, - "attention_head_dim": 8, - "in_channels": 4, - "out_channels": 4, - "text_dim": 32, - "freq_dim": 16, - "ffn_dim": 16, - "num_layers": 2, - "cross_attn_norm": True, - "qk_norm": "rms_norm_across_heads", - "rope_max_seq_len": 16, - "vace_layers": [0], - "vace_in_channels": 72, - } - transformer_cls = WanVACETransformer3DModel - vae_kwargs = { - "base_dim": 3, - "z_dim": 4, - "dim_mult": [1, 1, 1, 1], - "latents_mean": [-0.7571, -0.7089, -0.9113, -0.7245], - "latents_std": [2.8184, 1.4541, 2.3275, 2.6558], - "num_res_blocks": 1, - "temperal_downsample": [False, True, True], - } - vae_cls = AutoencoderKLWan - has_two_text_encoders = True - tokenizer_cls, tokenizer_id = AutoTokenizer, "hf-internal-testing/tiny-random-t5" - text_encoder_cls, text_encoder_id = T5EncoderModel, "hf-internal-testing/tiny-random-t5" - - text_encoder_target_modules = ["q", "k", "v", "o"] - - supports_text_encoder_loras = False - - @property - def output_shape(self): - return (1, 9, 16, 16, 3) - - def get_dummy_inputs(self, with_generator=True): - batch_size = 1 - sequence_length = 16 - num_channels = 4 - num_frames = 9 - num_latent_frames = 3 # (num_frames - 1) // temporal_compression_ratio + 1 - sizes = (4, 4) - height, width = 16, 16 - - generator = torch.manual_seed(0) - noise = floats_tensor((batch_size, num_latent_frames, num_channels) + sizes) - input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) - video = [Image.new("RGB", (height, width))] * num_frames - mask = [Image.new("L", (height, width), 0)] * num_frames - - pipeline_inputs = { - "video": video, - "mask": mask, - "prompt": "", - "num_frames": num_frames, - "num_inference_steps": 1, - "guidance_scale": 6.0, - "height": height, - "width": height, - "max_sequence_length": sequence_length, - "output_type": "np", - } - if with_generator: - pipeline_inputs.update({"generator": generator}) - - return noise, input_ids, pipeline_inputs - - def test_simple_inference_with_text_lora_denoiser_fused_multi(self): - super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3) - - def test_simple_inference_with_text_denoiser_lora_unfused(self): - super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3) - - @unittest.skip("Not supported in Wan VACE.") - def test_simple_inference_with_text_denoiser_block_scale(self): - pass - - @unittest.skip("Not supported in Wan VACE.") - def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self): - pass - - @require_peft_version_greater("0.13.2") - def test_lora_exclude_modules_wanvace(self): - exclude_module_name = "vace_blocks.0.proj_out" - components, text_lora_config, denoiser_lora_config = self.get_dummy_components() - pipe = self.pipeline_class(**components).to(torch_device) - _, _, inputs = self.get_dummy_inputs(with_generator=False) - - output_no_lora = self.get_base_pipe_output() - self.assertTrue(output_no_lora.shape == self.output_shape) - - # only supported for `denoiser` now - denoiser_lora_config.target_modules = ["proj_out"] - denoiser_lora_config.exclude_modules = [exclude_module_name] - pipe, _ = self.add_adapters_to_pipeline( - pipe, text_lora_config=text_lora_config, denoiser_lora_config=denoiser_lora_config - ) - # The state dict shouldn't contain the modules to be excluded from LoRA. - state_dict_from_model = get_peft_model_state_dict(pipe.transformer, adapter_name="default") - self.assertTrue(not any(exclude_module_name in k for k in state_dict_from_model)) - self.assertTrue(any("proj_out" in k for k in state_dict_from_model)) - output_lora_exclude_modules = pipe(**inputs, generator=torch.manual_seed(0))[0] - - with tempfile.TemporaryDirectory() as tmpdir: - modules_to_save = self._get_modules_to_save(pipe, has_denoiser=True) - lora_state_dicts = self._get_lora_state_dicts(modules_to_save) - self.pipeline_class.save_lora_weights(save_directory=tmpdir, **lora_state_dicts) - pipe.unload_lora_weights() - - # Check in the loaded state dict. - loaded_state_dict = safetensors.torch.load_file(os.path.join(tmpdir, "pytorch_lora_weights.safetensors")) - self.assertTrue(not any(exclude_module_name in k for k in loaded_state_dict)) - self.assertTrue(any("proj_out" in k for k in loaded_state_dict)) - - # Check in the state dict obtained after loading LoRA. - pipe.load_lora_weights(tmpdir) - state_dict_from_model = get_peft_model_state_dict(pipe.transformer, adapter_name="default_0") - self.assertTrue(not any(exclude_module_name in k for k in state_dict_from_model)) - self.assertTrue(any("proj_out" in k for k in state_dict_from_model)) - - output_lora_pretrained = pipe(**inputs, generator=torch.manual_seed(0))[0] - self.assertTrue( - not np.allclose(output_no_lora, output_lora_exclude_modules, atol=1e-3, rtol=1e-3), - "LoRA should change outputs.", - ) - self.assertTrue( - np.allclose(output_lora_exclude_modules, output_lora_pretrained, atol=1e-3, rtol=1e-3), - "Lora outputs should match.", - ) - - def test_simple_inference_with_text_denoiser_lora_and_scale(self): - super().test_simple_inference_with_text_denoiser_lora_and_scale() diff --git a/tests/lora/test_lora_loader_utils.py b/tests/lora/test_lora_loader_utils.py index ee0d5219cd8a..d79fd77089b0 100644 --- a/tests/lora/test_lora_loader_utils.py +++ b/tests/lora/test_lora_loader_utils.py @@ -27,7 +27,7 @@ from diffusers.models.modeling_utils import ModelMixin from diffusers.utils.import_utils import is_peft_available -from ..testing_utils import require_peft_backend +from ..testing_utils import CaptureLogger, require_peft_backend if is_peft_available(): @@ -99,20 +99,23 @@ def test_local_directory_without_matching_files_returns_none(tmp_path, monkeypat assert weight_name is None -def test_local_directory_with_multiple_files_warns_and_uses_first(tmp_path, monkeypatch, caplog): +def test_local_directory_with_multiple_files_warns_and_uses_first(tmp_path, monkeypatch): first_path = tmp_path / "first.safetensors" second_path = tmp_path / "second.safetensors" first_path.touch() second_path.touch() monkeypatch.setattr(lora_base, "HF_HUB_OFFLINE", True) + # `os.listdir` returns entries in arbitrary order; pin it so `first.safetensors` is the one picked. monkeypatch.setattr(lora_base.os, "listdir", lambda _: [first_path.name, second_path.name]) - monkeypatch.setattr(lora_base.logger, "propagate", True) - with caplog.at_level(logging.WARNING, logger="diffusers.loaders.lora_base"): + # `caplog` cannot see this warning: `diffusers` sets `propagate = False` on its library root logger, so + # records never reach the root handler pytest installs. `CaptureLogger` attaches to the logger directly. + lora_base.logger.setLevel(logging.WARNING) + with CaptureLogger(lora_base.logger) as cap_logger: weight_name = lora_base._best_guess_weight_name(tmp_path) assert weight_name == first_path.name - assert "contains more than one weights file" in caplog.text + assert "contains more than one weights file" in cap_logger.out @require_peft_backend diff --git a/tests/lora/utils.py b/tests/lora/utils.py index 8abe9242a6a7..dacfd61cc9f0 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -35,12 +35,10 @@ CaptureLogger, check_if_dicts_are_equal, floats_tensor, - is_torch_version, require_peft_backend, require_peft_version_greater, require_torch_accelerator, require_transformers_version_greater, - skip_mps, torch_device, ) @@ -1333,12 +1331,6 @@ def test_simple_inference_with_text_denoiser_multi_adapter_weighted(self): "output with no lora and output with lora disabled should give same results", ) - @skip_mps - @pytest.mark.xfail( - condition=torch.device(torch_device).type == "cpu" and is_torch_version(">=", "2.5"), - reason="Test currently fails on CPU and PyTorch 2.5.1 but not on PyTorch 2.4.1.", - strict=False, - ) def test_get_adapters(self): """ Tests a simple usecase where we attach multiple adapters and check if the results @@ -1693,16 +1685,6 @@ def test_set_adapters_match_attention_kwargs(self): "Loading from saved checkpoints should give same results as set_adapters().", ) - @pytest.mark.xfail( - condition=torch_device == "mps", - reason="MPS does not support float8 casting.", - strict=True, - ) - @pytest.mark.xfail( - condition=torch_device == "mps", - reason="MPS does not support float8 casting.", - strict=True, - ) @parameterized.expand([4, 8, 16]) def test_lora_adapter_metadata_is_loaded_correctly(self, lora_alpha): components, text_lora_config, denoiser_lora_config = self.get_dummy_components(lora_alpha=lora_alpha) diff --git a/tests/pipelines/cogvideo/test_cogvideox.py b/tests/pipelines/cogvideo/test_cogvideox.py index 5bc34896e194..82db4dba98a6 100644 --- a/tests/pipelines/cogvideo/test_cogvideox.py +++ b/tests/pipelines/cogvideo/test_cogvideox.py @@ -32,6 +32,8 @@ BasePipelineTesterConfig, FasterCacheTesterMixin, FirstBlockCacheTesterMixin, + LoraMemoryTesterMixin, + LoraTesterMixin, MemoryTesterMixin, PipelineTesterMixin, PyramidAttentionBroadcastTesterMixin, @@ -236,6 +238,21 @@ class TestCogVideoXPipelineFirstBlockCache(CogVideoXPipelineTesterConfig, FirstB pass +class TestCogVideoXPipelineLoRA(CogVideoXPipelineTesterConfig, LoraTesterMixin): + """LoRA tests for the CogVideoX pipeline.""" + + +class TestCogVideoXPipelineLoRAMemory(CogVideoXPipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the CogVideoX pipeline.""" + + # `(leaf_level, True)` is left out on purpose, see + # https://github.com/huggingface/diffusers/pull/11804#issuecomment-3013325338 + @pytest.mark.parametrize("offload_type,use_stream", [("block_level", True), ("leaf_level", False)]) + @require_torch_accelerator + def test_group_offloading_inference_denoiser(self, tmp_path, offload_type, use_stream): + super().test_group_offloading_inference_denoiser(tmp_path, offload_type, use_stream) + + @nightly @require_torch_accelerator class TestCogVideoXPipelineIntegration: diff --git a/tests/pipelines/flux2/test_pipeline_flux2.py b/tests/pipelines/flux2/test_pipeline_flux2.py index 73c480a82809..c0ea0292f808 100644 --- a/tests/pipelines/flux2/test_pipeline_flux2.py +++ b/tests/pipelines/flux2/test_pipeline_flux2.py @@ -11,6 +11,8 @@ from ...testing_utils import assert_tensors_close, torch_device from ..testing_utils import ( BasePipelineTesterConfig, + LoraMemoryTesterMixin, + LoraTesterMixin, MemoryTesterMixin, PipelineTesterMixin, check_qkv_fused_layers_exist, @@ -187,3 +189,16 @@ def test_flux_image_output_shape(self): class TestFlux2PipelineMemory(Flux2PipelineTesterConfig, MemoryTesterMixin): """Memory optimization tests (CPU offload, group offload, layerwise casting) for the Flux2 pipeline.""" + + +class TestFlux2PipelineLoRA(Flux2PipelineTesterConfig, LoraTesterMixin): + """LoRA tests for the Flux2 pipeline.""" + + # Flux2 fuses the QKV and MLP input projections into a single `to_qkv_mlp_proj` linear. + denoiser_target_modules = {"transformer": ["to_qkv_mlp_proj", "to_k"]} + + +class TestFlux2PipelineLoRAMemory(Flux2PipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the Flux2 pipeline.""" + + denoiser_target_modules = {"transformer": ["to_qkv_mlp_proj", "to_k"]} diff --git a/tests/pipelines/qwenimage/test_qwenimage.py b/tests/pipelines/qwenimage/test_qwenimage.py index 989862c52c90..44e3d87a18e9 100644 --- a/tests/pipelines/qwenimage/test_qwenimage.py +++ b/tests/pipelines/qwenimage/test_qwenimage.py @@ -25,6 +25,8 @@ from ...testing_utils import assert_tensors_close, torch_device from ..testing_utils import ( BasePipelineTesterConfig, + LoraMemoryTesterMixin, + LoraTesterMixin, MemoryTesterMixin, PipelineTesterMixin, ) @@ -188,3 +190,11 @@ def test_true_cfg_without_negative_prompt_embeds_mask(self): class TestQwenImagePipelineMemory(QwenImagePipelineTesterConfig, MemoryTesterMixin): pass + + +class TestQwenImagePipelineLoRA(QwenImagePipelineTesterConfig, LoraTesterMixin): + """LoRA tests for the QwenImage pipeline.""" + + +class TestQwenImagePipelineLoRAMemory(QwenImagePipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the QwenImage pipeline.""" diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index c8de618d6c11..87ee7100a73c 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -20,7 +20,9 @@ import numpy as np import pytest import torch +import torch.nn as nn from huggingface_hub import hf_hub_download +from safetensors.torch import load_file from transformers import ( CLIPTextConfig, CLIPTextModel, @@ -29,7 +31,10 @@ from diffusers import ( AutoencoderKL, + AutoPipelineForImage2Image, + AutoPipelineForText2Image, DDIMScheduler, + DiffusionPipeline, DPMSolverMultistepScheduler, EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, @@ -40,21 +45,27 @@ UNet2DConditionModel, logging, ) +from diffusers.utils.import_utils import is_accelerate_available +from ...models.testing_utils.lora import check_if_lora_correctly_set from ...testing_utils import ( CaptureLogger, + Expectations, assert_tensors_close, backend_empty_cache, backend_max_memory_allocated, backend_reset_max_memory_allocated, backend_reset_peak_memory_stats, + load_image, load_numpy, nightly, numpy_cosine_similarity_distance, require_accelerate_version_greater, + require_peft_backend, require_torch_accelerator, require_torch_multi_accelerator, skip_mps, + slow, torch_device, ) from ..pipeline_params import ( @@ -63,12 +74,19 @@ ) from ..testing_utils import ( BasePipelineTesterConfig, + LoraMemoryTesterMixin, + LoraTesterMixin, MemoryTesterMixin, PipelineTesterMixin, + UNetLoraTesterMixin, ) from .ip_adapter_tester import IPAdapterTesterMixin +if is_accelerate_available(): + from accelerate.utils import release_memory + + class StableDiffusionPipelineTesterConfig(BasePipelineTesterConfig): pipeline_class = StableDiffusionPipeline required_input_params_in_call_signature = TEXT_TO_IMAGE_PARAMS @@ -605,6 +623,14 @@ class TestStableDiffusionPipelineMemory(StableDiffusionPipelineTesterConfig, Mem """Memory optimization tests (CPU offload, group offload, layerwise casting) for the Stable Diffusion pipeline.""" +class TestStableDiffusionPipelineLoRA(StableDiffusionPipelineTesterConfig, LoraTesterMixin, UNetLoraTesterMixin): + """LoRA tests for the Stable Diffusion pipeline.""" + + +class TestStableDiffusionPipelineLoRAMemory(StableDiffusionPipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the Stable Diffusion pipeline.""" + + class TestStableDiffusionPipelineIPAdapter(StableDiffusionPipelineTesterConfig, IPAdapterTesterMixin): """IP-Adapter tests for the Stable Diffusion pipeline.""" @@ -1294,3 +1320,654 @@ def test_reset_device_map_enable_sequential_cpu_offload(self): # Make sure `enable_sequential_cpu_offload()` can be used and the pipeline can be called. sd_pipe_with_device_map.enable_sequential_cpu_offload(device=torch_device) _ = sd_pipe_with_device_map("hello", num_inference_steps=2) + + +@slow +@require_torch_accelerator +@require_peft_backend +class TestStableDiffusionLoRADeviceIntegration: + """`set_lora_device` behavior on the real SD 1.5 checkpoint (no logit assertions).""" + + @pytest.fixture(autouse=True) + def cleanup(self): + gc.collect() + backend_empty_cache(torch_device) + yield + gc.collect() + backend_empty_cache(torch_device) + + def test_integration_move_lora_cpu(self): + path = "stable-diffusion-v1-5/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) + pipe.load_lora_weights(lora_id, adapter_name="adapter-1") + pipe.load_lora_weights(lora_id, adapter_name="adapter-2") + pipe = pipe.to(torch_device) + + assert check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + + assert check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in unet" + + # We will offload the first adapter in CPU and check if the offloading + # has been performed correctly + pipe.set_lora_device(["adapter-1"], "cpu") + + for name, module in pipe.unet.named_modules(): + if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device == torch.device("cpu") + elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device != torch.device("cpu") + + for name, module in pipe.text_encoder.named_modules(): + if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device == torch.device("cpu") + elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device != torch.device("cpu") + + pipe.set_lora_device(["adapter-1"], 0) + + for n, m in pipe.unet.named_modules(): + if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): + assert m.weight.device != torch.device("cpu") + + for n, m in pipe.text_encoder.named_modules(): + if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): + assert m.weight.device != torch.device("cpu") + + pipe.set_lora_device(["adapter-1", "adapter-2"], torch_device) + + for n, m in pipe.unet.named_modules(): + if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): + assert m.weight.device != torch.device("cpu") + + for n, m in pipe.text_encoder.named_modules(): + if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): + assert m.weight.device != torch.device("cpu") + + def test_integration_move_lora_dora_cpu(self): + from peft import LoraConfig + + path = "stable-diffusion-v1-5/stable-diffusion-v1-5" + unet_lora_config = LoraConfig( + init_lora_weights="gaussian", + target_modules=["to_k", "to_q", "to_v", "to_out.0"], + use_dora=True, + ) + text_lora_config = LoraConfig( + init_lora_weights="gaussian", + target_modules=["q_proj", "k_proj", "v_proj", "out_proj"], + use_dora=True, + ) + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + + assert check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + + assert check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in unet" + + for name, param in pipe.unet.named_parameters(): + if "lora_" in name: + assert param.device == torch.device("cpu") + + for name, param in pipe.text_encoder.named_parameters(): + if "lora_" in name: + assert param.device == torch.device("cpu") + + pipe.set_lora_device(["adapter-1"], torch_device) + + for name, param in pipe.unet.named_parameters(): + if "lora_" in name: + assert param.device != torch.device("cpu") + + for name, param in pipe.text_encoder.named_parameters(): + if "lora_" in name: + assert param.device != torch.device("cpu") + + def test_integration_set_lora_device_different_target_layers(self): + # fixes a bug that occurred when calling set_lora_device with multiple adapters loaded that target different + # layers, see #11833 + from peft import LoraConfig + + path = "stable-diffusion-v1-5/stable-diffusion-v1-5" + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) + # configs partly target the same, partly different layers + config0 = LoraConfig(target_modules=["to_k", "to_v"]) + config1 = LoraConfig(target_modules=["to_k", "to_q"]) + pipe.unet.add_adapter(config0, adapter_name="adapter-0") + pipe.unet.add_adapter(config1, adapter_name="adapter-1") + pipe = pipe.to(torch_device) + + assert check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in unet" + + # sanity check that the adapters don't target the same layers, otherwise the test passes even without the fix + modules_adapter_0 = {n for n, _ in pipe.unet.named_modules() if n.endswith(".adapter-0")} + modules_adapter_1 = {n for n, _ in pipe.unet.named_modules() if n.endswith(".adapter-1")} + assert modules_adapter_0 != modules_adapter_1 + assert modules_adapter_0 - modules_adapter_1 + assert modules_adapter_1 - modules_adapter_0 + + # setting both separately works + pipe.set_lora_device(["adapter-0"], "cpu") + pipe.set_lora_device(["adapter-1"], "cpu") + + for name, module in pipe.unet.named_modules(): + if "adapter-0" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device == torch.device("cpu") + elif "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device == torch.device("cpu") + + # setting both at once also works + pipe.set_lora_device(["adapter-0", "adapter-1"], torch_device) + + for name, module in pipe.unet.named_modules(): + if "adapter-0" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device != torch.device("cpu") + elif "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + assert module.weight.device != torch.device("cpu") + + +@slow +@nightly +@require_torch_accelerator +@require_peft_backend +class TestStableDiffusionLoRAIntegration: + @pytest.fixture(autouse=True) + def cleanup(self): + gc.collect() + backend_empty_cache(torch_device) + yield + gc.collect() + backend_empty_cache(torch_device) + + def test_integration_logits_with_scale(self): + path = "stable-diffusion-v1-5/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) + pipe.load_lora_weights(lora_id) + pipe = pipe.to(torch_device) + + assert check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + + prompt = "a red sks dog" + + images = pipe( + prompt=prompt, + num_inference_steps=15, + cross_attention_kwargs={"scale": 0.5}, + generator=torch.manual_seed(0), + output_type="np", + ).images + + expected_slice_scale = np.array([0.307, 0.283, 0.310, 0.310, 0.300, 0.314, 0.336, 0.314, 0.321]) + predicted_slice = images[0, -3:, -3:, -1].flatten() + + max_diff = numpy_cosine_similarity_distance(expected_slice_scale, predicted_slice) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_integration_logits_no_scale(self): + path = "stable-diffusion-v1-5/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) + pipe.load_lora_weights(lora_id) + pipe = pipe.to(torch_device) + + assert check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + + prompt = "a red sks dog" + + images = pipe(prompt=prompt, num_inference_steps=30, generator=torch.manual_seed(0), output_type="np").images + + expected_slice_scale = np.array([0.074, 0.064, 0.073, 0.0842, 0.069, 0.0641, 0.0794, 0.076, 0.084]) + predicted_slice = images[0, -3:, -3:, -1].flatten() + + max_diff = numpy_cosine_similarity_distance(expected_slice_scale, predicted_slice) + + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_dreambooth_old_format(self): + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "hf-internal-testing/lora_dreambooth_dog_example" + + base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe( + "A photo of a sks dog floating in the river", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.7207, 0.6787, 0.6010, 0.7478, 0.6838, 0.6064, 0.6984, 0.6443, 0.5785]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_dreambooth_text_encoder_new_format(self): + generator = torch.Generator().manual_seed(0) + + lora_model_id = "hf-internal-testing/lora-trained" + + base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe("A photo of a sks dog", output_type="np", generator=generator, num_inference_steps=2).images + + images = images[0, -3:, -3:, -1].flatten() + + expected = np.array([0.6628, 0.6138, 0.5390, 0.6625, 0.6130, 0.5463, 0.6166, 0.5788, 0.5359]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_a1111(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None).to( + torch_device + ) + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_lycoris(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/Amixx", safety_checker=None, use_safetensors=True, variant="fp16" + ).to(torch_device) + lora_model_id = "hf-internal-testing/edgLycorisMugler-light" + lora_filename = "edgLycorisMugler-light.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.6463, 0.658, 0.599, 0.6542, 0.6512, 0.6213, 0.658, 0.6485, 0.6017]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_a1111_with_model_cpu_offload(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) + pipe.enable_model_cpu_offload(device=torch_device) + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_a1111_with_sequential_cpu_offload(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) + pipe.enable_sequential_cpu_offload(device=torch_device) + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_kohya_sd_v15_with_higher_dimensions(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None + ).to(torch_device) + lora_model_id = "hf-internal-testing/urushisato-lora" + lora_filename = "urushisato_v15.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.7165, 0.6616, 0.5833, 0.7504, 0.6718, 0.587, 0.6871, 0.6361, 0.5694]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_vanilla_funetuning(self): + generator = torch.Generator().manual_seed(0) + + lora_model_id = "hf-internal-testing/sd-model-finetuned-lora-t4" + + base_model_id = "stable-diffusion-v1-5/stable-diffusion-v1-5" + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe("A pokemon with blue eyes.", output_type="np", generator=generator, num_inference_steps=2).images + + image_slice = images[0, -3:, -3:, -1].flatten() + + expected_slices = Expectations( + { + ("xpu", 3): np.array( + [ + 0.6544, + 0.6127, + 0.5397, + 0.6845, + 0.6047, + 0.5469, + 0.6349, + 0.5906, + 0.5382, + ] + ), + ("cuda", 7): np.array( + [ + 0.7406, + 0.699, + 0.5963, + 0.7493, + 0.7045, + 0.6096, + 0.6886, + 0.6388, + 0.583, + ] + ), + ("cuda", 8): np.array( + [ + 0.6542, + 0.61253, + 0.5396, + 0.6843, + 0.6044, + 0.5468, + 0.6349, + 0.5905, + 0.5381, + ] + ), + } + ) + expected_slice = expected_slices.get_expectation() + + max_diff = numpy_cosine_similarity_distance(expected_slice, image_slice) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + release_memory(pipe) + + def test_unload_kohya_lora(self): + generator = torch.manual_seed(0) + prompt = "masterpiece, best quality, mountain" + num_inference_steps = 2 + + pipe = StableDiffusionPipeline.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None + ).to(torch_device) + initial_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + initial_images = initial_images[0, -3:, -3:, -1].flatten() + + lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" + lora_filename = "Colored_Icons_by_vizsumit.safetensors" + + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images = lora_images[0, -3:, -3:, -1].flatten() + + pipe.unload_lora_weights() + generator = torch.manual_seed(0) + unloaded_lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() + + assert not np.allclose(initial_images, lora_images) + assert np.allclose(initial_images, unloaded_lora_images, atol=1e-3) + + release_memory(pipe) + + def test_load_unload_load_kohya_lora(self): + # This test ensures that a Kohya-style LoRA can be safely unloaded and then loaded + # without introducing any side-effects. Even though the test uses a Kohya-style + # LoRA, the underlying adapter handling mechanism is format-agnostic. + generator = torch.manual_seed(0) + prompt = "masterpiece, best quality, mountain" + num_inference_steps = 2 + + pipe = StableDiffusionPipeline.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", safety_checker=None + ).to(torch_device) + initial_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + initial_images = initial_images[0, -3:, -3:, -1].flatten() + + lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" + lora_filename = "Colored_Icons_by_vizsumit.safetensors" + + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images = lora_images[0, -3:, -3:, -1].flatten() + + pipe.unload_lora_weights() + generator = torch.manual_seed(0) + unloaded_lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() + + assert not np.allclose(initial_images, lora_images) + assert np.allclose(initial_images, unloaded_lora_images, atol=1e-3) + + # make sure we can load a LoRA again after unloading and they don't have + # any undesired effects. + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images_again = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images_again = lora_images_again[0, -3:, -3:, -1].flatten() + + assert np.allclose(lora_images, lora_images_again, atol=1e-3) + release_memory(pipe) + + def test_not_empty_state_dict(self): + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again + pipe = AutoPipelineForText2Image.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 + ).to(torch_device) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") + lcm_lora = load_file(cached_file) + + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + assert lcm_lora != {} + release_memory(pipe) + + def test_load_unload_load_state_dict(self): + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again + pipe = AutoPipelineForText2Image.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 + ).to(torch_device) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") + lcm_lora = load_file(cached_file) + previous_state_dict = lcm_lora.copy() + + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + assert lcm_lora == previous_state_dict + + pipe.unload_lora_weights() + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + assert lcm_lora == previous_state_dict + + release_memory(pipe) + + def test_sdv1_5_lcm_lora(self): + pipe = DiffusionPipeline.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 + ) + pipe.to(torch_device) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "latent-consistency/lcm-lora-sdv1-5" + pipe.load_lora_weights(lora_model_id) + + image = pipe( + "masterpiece, best quality, mountain", generator=generator, num_inference_steps=4, guidance_scale=0.5 + ).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora.png" + ) + + image_np = pipe.image_processor.pil_to_numpy(image) + expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + + release_memory(pipe) + + def test_sdv1_5_lcm_lora_img2img(self): + pipe = AutoPipelineForImage2Image.from_pretrained( + "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 + ) + pipe.to(torch_device) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape.png" + ) + + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "latent-consistency/lcm-lora-sdv1-5" + pipe.load_lora_weights(lora_model_id) + + image = pipe( + "snowy mountain", + generator=generator, + image=init_image, + strength=0.5, + num_inference_steps=4, + guidance_scale=0.5, + ).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora_img2img.png" + ) + + image_np = pipe.image_processor.pil_to_numpy(image) + expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + + release_memory(pipe) + + def test_sd_load_civitai_empty_network_alpha(self): + """ + This test simply checks that loading a LoRA with an empty network alpha works fine + See: https://github.com/huggingface/diffusers/issues/5606 + """ + pipeline = StableDiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") + pipeline.enable_sequential_cpu_offload(device=torch_device) + civitai_path = hf_hub_download("ybelkada/test-ahi-civitai", "ahi_lora_weights.safetensors") + pipeline.load_lora_weights(civitai_path, adapter_name="ahri") + + images = pipeline( + "ahri, masterpiece, league of legends", + output_type="np", + generator=torch.manual_seed(156), + num_inference_steps=5, + ).images + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.0, 0.0, 0.0, 0.002557, 0.020954, 0.001792, 0.006581, 0.00591, 0.002995]) + + max_diff = numpy_cosine_similarity_distance(expected, images) + assert max_diff < 1e-3 + + pipeline.unload_lora_weights() + release_memory(pipeline) diff --git a/tests/pipelines/wan/test_wan.py b/tests/pipelines/wan/test_wan.py index de4d1a6654b1..2815e5657e8a 100644 --- a/tests/pipelines/wan/test_wan.py +++ b/tests/pipelines/wan/test_wan.py @@ -19,7 +19,13 @@ from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanPipeline, WanTransformer3DModel from ...testing_utils import assert_tensors_close, torch_device -from ..testing_utils import BasePipelineTesterConfig, MemoryTesterMixin, PipelineTesterMixin +from ..testing_utils import ( + BasePipelineTesterConfig, + LoraMemoryTesterMixin, + LoraTesterMixin, + MemoryTesterMixin, + PipelineTesterMixin, +) class WanPipelineTesterConfig(BasePipelineTesterConfig): @@ -142,3 +148,11 @@ def test_save_load_optional_components(self, tmp_path, expected_max_difference=1 class TestWanPipelineMemory(WanPipelineTesterConfig, MemoryTesterMixin): pass + + +class TestWanPipelineLoRA(WanPipelineTesterConfig, LoraTesterMixin): + """LoRA tests for the Wan pipeline.""" + + +class TestWanPipelineLoRAMemory(WanPipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the Wan pipeline.""" diff --git a/tests/pipelines/wan/test_wan_vace.py b/tests/pipelines/wan/test_wan_vace.py index 6c4e82521196..7bff98bc647a 100644 --- a/tests/pipelines/wan/test_wan_vace.py +++ b/tests/pipelines/wan/test_wan_vace.py @@ -13,7 +13,10 @@ # limitations under the License. +import os + import pytest +import safetensors.torch import torch from PIL import Image from transformers import AutoConfig, AutoTokenizer, T5EncoderModel @@ -25,9 +28,20 @@ WanVACEPipeline, WanVACETransformer3DModel, ) +from diffusers.utils.import_utils import is_peft_available + +from ...testing_utils import assert_tensors_close, require_peft_version_greater, torch_device +from ..testing_utils import ( + BasePipelineTesterConfig, + LoraMemoryTesterMixin, + LoraTesterMixin, + MemoryTesterMixin, + PipelineTesterMixin, +) + -from ...testing_utils import assert_tensors_close, torch_device -from ..testing_utils import BasePipelineTesterConfig, MemoryTesterMixin, PipelineTesterMixin +if is_peft_available(): + from peft.utils import get_peft_model_state_dict class WanVACEPipelineTesterConfig(BasePipelineTesterConfig): @@ -242,3 +256,54 @@ def test_save_load_optional_components(self, tmp_path, expected_max_difference=1 class TestWanVACEPipelineMemory(WanVACEPipelineTesterConfig, MemoryTesterMixin): pass + + +class TestWanVACEPipelineLoRA(WanVACEPipelineTesterConfig, LoraTesterMixin): + """LoRA tests for the Wan VACE pipeline.""" + + @require_peft_version_greater("0.13.2") + def test_lora_exclude_modules(self, tmp_path, base_pipe_output): + """`exclude_modules` must keep the excluded module out of the adapter and out of every state dict.""" + exclude_module_name = "vace_blocks.0.proj_out" + + pipe = self.get_pipeline().to(torch_device) + adapted = self.add_adapters_to_pipeline( + pipe, + components=self.denoiser_components, + target_modules=["proj_out"], + exclude_modules=[exclude_module_name], + ) + + # The state dict shouldn't contain the modules to be excluded from LoRA. + state_dict_from_model = get_peft_model_state_dict(pipe.transformer, adapter_name="default") + assert not any(exclude_module_name in k for k in state_dict_from_model) + assert any("proj_out" in k for k in state_dict_from_model) + + output_lora_exclude_modules = self.run_pipe(pipe) + + lora_state_dicts = self._get_lora_state_dicts(adapted) + self.pipeline_class.save_lora_weights(save_directory=tmp_path, **lora_state_dicts) + pipe.unload_lora_weights() + + # Check in the loaded state dict. + loaded_state_dict = safetensors.torch.load_file(os.path.join(tmp_path, "pytorch_lora_weights.safetensors")) + assert not any(exclude_module_name in k for k in loaded_state_dict) + assert any("proj_out" in k for k in loaded_state_dict) + + # Check in the state dict obtained after loading LoRA. + pipe.load_lora_weights(tmp_path) + state_dict_from_model = get_peft_model_state_dict(pipe.transformer, adapter_name="default_0") + assert not any(exclude_module_name in k for k in state_dict_from_model) + assert any("proj_out" in k for k in state_dict_from_model) + + output_lora_pretrained = self.run_pipe(pipe) + assert not torch.allclose(base_pipe_output, output_lora_exclude_modules, atol=1e-3, rtol=1e-3), ( + "LoRA should change outputs." + ) + assert_tensors_close( + output_lora_pretrained, output_lora_exclude_modules, atol=1e-3, rtol=1e-3, msg="Lora outputs should match." + ) + + +class TestWanVACEPipelineLoRAMemory(WanVACEPipelineTesterConfig, LoraMemoryTesterMixin): + """LoRA x memory-optimization tests for the Wan VACE pipeline."""