Lora not functioning when used with t2i adapters Pipeline

[ilisparrow]

Describe the bug

Hello,

Thank you for this useful library.
I have a small problem, I managed to use the code to generate images using SDXL with t2i, then an Image with a Lora. But for some reasons not with both.
When I load the lora using the StableDiffusionXLPipeline It does work but not when I do so with this : StablediffusionXLPipeline
I already had a similar problem with controlnet and Lora. Here is the issue : #4664.
I followed somehow similar steps, and managed to fix it on my local install. I have added this bit of code at the end of pipeline_stable_diffusion_xl_adapter (Without forgetting the import os) :

# Overrride to properly handle the loading and unloading of the additional text encoder.

    def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs):
        # We could have accessed the unet config from `lora_state_dict()` too. We pass
        # it here explicitly to be able to tell that it's coming from an SDXL
        # pipeline.
        state_dict, network_alphas = self.lora_state_dict(
            pretrained_model_name_or_path_or_dict,
            unet_config=self.unet.config,
            **kwargs,
        )
        self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet)

        text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k}
        if len(text_encoder_state_dict) > 0:
            self.load_lora_into_text_encoder(
                text_encoder_state_dict,
                network_alphas=network_alphas,
                text_encoder=self.text_encoder,
                prefix="text_encoder",
                lora_scale=self.lora_scale,
            )

        text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k}
        if len(text_encoder_2_state_dict) > 0:
            self.load_lora_into_text_encoder(
                text_encoder_2_state_dict,
                network_alphas=network_alphas,
                text_encoder=self.text_encoder_2,
                prefix="text_encoder_2",
                lora_scale=self.lora_scale,
            )

    @classmethod

    def save_lora_weights(
        self,
        save_directory: Union[str, os.PathLike],
        unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None,
        text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None,
        text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None,
        is_main_process: bool = True,
        weight_name: str = None,
        save_function: Callable = None,
        safe_serialization: bool = True,
    ):
        state_dict = {}

        def pack_weights(layers, prefix):
            layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers
            layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()}
            return layers_state_dict

        state_dict.update(pack_weights(unet_lora_layers, "unet"))

        if text_encoder_lora_layers and text_encoder_2_lora_layers:
            state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder"))
            state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2"))

        self.write_lora_layers(
            state_dict=state_dict,
            save_directory=save_directory,
            is_main_process=is_main_process,
            weight_name=weight_name,
            save_function=save_function,
            safe_serialization=safe_serialization,
        )
diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._remove_text_encoder_monkey_patch
    def _remove_text_encoder_monkey_patch(self):
        self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder)
        self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2)

I would love to open a PR and contribute to your code, the files don't look exactly the same (the one I have installed, and the one I cloned).

And I hope that this will help improve the library :).

Reproduction

import os
import datetime
from diffusers import StableDiffusionXLAdapterPipeline, T2IAdapter, EulerAncestralDiscreteScheduler, AutoencoderKL
from diffusers.utils import load_image, make_image_grid
from controlnet_aux.lineart import LineartDetector
import torch



def save_image_with_timestamp(image, output_folder='output/'):
    # Ensure the output folder exists, creating it if necessary
    os.makedirs(output_folder, exist_ok=True)

    # Get the current date and time
    current_time = datetime.datetime.now()

    # Format the date and time as a string
    timestamp = current_time.strftime("%Y-%m-%d_%H-%M-%S")

    # Create the filename using the timestamp
    filename = f"{timestamp}.png"

    # Combine the output folder and filename to get the full output path
    output_path = os.path.join(output_folder, filename)

    # Save the image to the specified output path
    image.save(output_path)

    # Return the full path of the saved image
    return output_path







# load adapter
adapter = T2IAdapter.from_pretrained(
  "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16, varient="fp16"
).to("cuda")

# load euler_a scheduler
model_id = 'stabilityai/stable-diffusion-xl-base-1.0'
euler_a = EulerAncestralDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
vae=AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
    model_id, vae=vae, adapter=adapter, scheduler=euler_a, torch_dtype=torch.float16, variant="fp16", 
).to("cuda")

pipe.load_lora_weights("lora/EnvyAnimeOilXL01.safetensors")
#pipe.load_lora_weights("lora/EnvyAnimeOilXL01.safetensors", weight_name="LogoRedmond_LogoRedAF.safetensors")

pipe.enable_xformers_memory_efficient_attention()

line_detector = LineartDetector.from_pretrained("lllyasviel/Annotators").to("cuda")
url = "https://huggingface.co/Adapter/t2iadapter/resolve/main/figs_SDXLV1.0/org_lin.jpg"

image = load_image(url)
image = line_detector(
    image, detect_resolution=384, image_resolution=1024
)
prompt = "Painting of a dragon, van gogh"
negative_prompt = "anime, cartoon, graphic, text, painting, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured"
gen_images = pipe(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=image,
    num_inference_steps=30,
    adapter_conditioning_scale=0.8,
    guidance_scale=7.5,
).images
for gen_image in gen_images : 
    save_image_with_timestamp(gen_image, output_folder='output/')

Logs

===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please run

python -m bitsandbytes

 and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
================================================================================
bin /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cextension.py:34: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.
  warn("The installed version of bitsandbytes was compiled without GPU support. "
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so: undefined symbol: cadam32bit_grad_fp32
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /home/user/anaconda3/envs/sandbox did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/ubuntu.mandatory.path')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/ubuntu.default.path')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('@/tmp/.ICE-unix/2636,unix/user-Desktop'), PosixPath('local/user-Desktop')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/org/gnome/Terminal/screen/438bc017_a4a2_48a9_bb96_5fdaaadd8000')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('0'), PosixPath('1')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/etc/xdg/xdg-ubuntu')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('CONDA_DEFAULT_ENV=sandbox'), PosixPath('CONDA_PROMPT_MODIFIER=(sandbox) '), PosixPath('CONDA_PREFIX_1=/home/user/anaconda3'), PosixPath('_CE_M='), PosixPath('_CE_CONDA='), PosixPath('CONDA_PREFIX=/home/user/anaconda3/envs/sandbox'), PosixPath('CONDA_SHLVL=2'), PosixPath('CONDA_ROOT=/home/user/anaconda3')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('PS1=(sandbox) '), PosixPath('PATH=/home/user/anaconda3/envs/sandbox/bin\\x3a/home/user/.local/bin\\x3a/home/user/anaconda3/condabin\\x3a/home/user/.cargo/bin\\x3a/usr/local/sbin\\x3a/usr/local/bin\\x3a/usr/sbin\\x3a/usr/bin\\x3a/sbin\\x3a/bin\\x3a/usr/games\\x3a/usr/local/games\\x3a/snap/bin\\x3a')}
  warn(msg)
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/cuda/lib64')}
  warn(msg)
/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!
  warn(msg)
CUDA SETUP: Highest compute capability among GPUs detected: 8.6
CUDA SETUP: Detected CUDA version 121
CUDA SETUP: Loading binary /home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cpu.so...
Loading pipeline components...: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:02<00:00,  2.79it/s]
Backend QtAgg is interactive backend. Turning interactive mode on.
Traceback (most recent call last):
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/runpy.py", line 197, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/__main__.py", line 39, in <module>
    cli.main()
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 430, in main
    run()
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/adapter/../../debugpy/launcher/../../debugpy/../debugpy/server/cli.py", line 284, in run_file
    runpy.run_path(target, run_name="__main__")
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 321, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 135, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "/home/user/.vscode/extensions/ms-python.python-2023.18.0/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_runpy.py", line 124, in _run_code
    exec(code, run_globals)
  File "/mnt/bigStorage/projects/SDXL/control.py", line 53, in <module>
    pipe.load_lora_weights("lora/EnvyAnimeOilXL01.safetensors")
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py", line 928, in load_lora_weights
    self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet)
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py", line 1210, in load_lora_into_unet
    unet.load_attn_procs(state_dict, network_alphas=network_alphas)
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/diffusers/loaders.py", line 366, in load_attn_procs
    attn_processor = getattr(attn_processor, sub_key)
  File "/home/user/anaconda3/envs/sandbox/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1695, in __getattr__
    raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
AttributeError: 'ModuleList' object has no attribute '4'

System Info

• diffusers version: 0.21.0.dev0
• Platform: Linux-6.2.0-35-generic-x86_64-with-glibc2.35
• Python version: 3.9.15
• PyTorch version (GPU?): 2.1.0+cu121 (True)
• Huggingface_hub version: 0.15.1
• Transformers version: 4.31.0
• Accelerate version: 0.21.0
• xFormers version: 0.0.22.post4
• Using GPU in script?: Yes
• Using distributed or parallel set-up in script?: No

Who can help?

@sayakpaul

[sayakpaul]

The issue is a bit convoluted. Can you provide a shorter code snippet and help me understand what's the problem?

#4664 is already solved. See here: #4664 (comment).

[ilisparrow]

Hello,
Sorry, indeed it was, because of the code markdown.
I fixed it.
Here is the code to reproduce it. You would need any Lora for SDXL inside a ./lora folder :

import os
import datetime
from diffusers import StableDiffusionXLAdapterPipeline, T2IAdapter, EulerAncestralDiscreteScheduler, AutoencoderKL
from diffusers.utils import load_image, make_image_grid
from controlnet_aux.lineart import LineartDetector
import torch

# load adapter
adapter = T2IAdapter.from_pretrained(
  "TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16, varient="fp16"
).to("cuda")

# load euler_a scheduler
model_id = 'stabilityai/stable-diffusion-xl-base-1.0'
euler_a = EulerAncestralDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
vae=AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
    model_id, vae=vae, adapter=adapter, scheduler=euler_a, torch_dtype=torch.float16, variant="fp16", 
).to("cuda")

pipe.load_lora_weights("lora/Samaritan 3d Cartoon SDXL.safetensors")

pipe.enable_xformers_memory_efficient_attention()

line_detector = LineartDetector.from_pretrained("lllyasviel/Annotators").to("cuda")
#url = "https://huggingface.co/Adapter/t2iadapter/resolve/main/figs_SDXLV1.0/org_lin.jpg"
url = "https://images.pexels.com/photos/18710356/pexels-photo-18710356/free-photo-of-woman-in-eyeglasses-in-black-and-white.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1"
image = load_image(url)
image = line_detector(
    image, detect_resolution=384, image_resolution=1024
)
prompt = "Painting of a woman, oil painting"#Standard
prompt = "a woman, Chinese ink painting,liujiyou"#"lora/liujiyou-SDXL.safetensors"
prompt = "a portrait painting of a woman, v0ng44g p0rtr14t"#"lora/v0ng44g, p0rtr14t.safetensors"
prompt = "a portrait of a woman wearing glasses, 3D, cartoon, pixar, disney "#"lora/Samaritan 3d Cartoon SDXL.safetensors"


negative_prompt = "text, crayon, graphite, abstract, glitch, deformed, mutated, ugly, disfigured"
gen_images = pipe(
    prompt=prompt,
    negative_prompt=negative_prompt,
    image=image,
    num_inference_steps=30,
    adapter_conditioning_scale=0.8,
    guidance_scale=7.5,
).images[0]
gen_images.save("example.jpg")

Best regards,

[sayakpaul]

I can reproduce it here: https://colab.research.google.com/gist/sayakpaul/1989dcb2428719e6be5850ade1c23eb6/scratchpad.ipynb

Feel free to open a PR :-)

Here is our contribution guidelines:
https://github.com/huggingface/diffusers/blob/main/CONTRIBUTING.md