From 8d267b5c50cf9474ffa7d53ab31f594c91a53d8a Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 12 Apr 2024 12:50:03 +0530 Subject: [PATCH 1/5] is_cosxl_edit arg in SDXL ip2p. --- src/diffusers/loaders/single_file.py | 6 +++++- .../pipeline_stable_diffusion_xl_instruct_pix2pix.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py index 752ef18c7a0b..5dc525bf9a15 100644 --- a/src/diffusers/loaders/single_file.py +++ b/src/diffusers/loaders/single_file.py @@ -136,6 +136,7 @@ def set_additional_components( original_config, checkpoint=None, model_type=None, + is_cosxl_edit=False, ): components = {} if pipeline_class_name in REFINER_PIPELINES: @@ -147,6 +148,8 @@ def set_additional_components( "force_zeros_for_empty_prompt": False if is_refiner else True, } ) + if pipeline_class_name == "StableDiffusionXLInstructPix2PixPipeline": + components.update({"is_cosxl_edit": is_cosxl_edit}) return components @@ -252,6 +255,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) torch_dtype = kwargs.pop("torch_dtype", None) + is_cosxl_edit = kwargs.pop("is_cosxl_edit", None) class_name = cls.__name__ @@ -309,7 +313,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): init_kwargs.update(components) additional_components = set_additional_components( - class_name, original_config, checkpoint=checkpoint, model_type=model_type + class_name, original_config, checkpoint=checkpoint, model_type=model_type, is_cosxl_edit=is_cosxl_edit ) if additional_components: init_kwargs.update(additional_components) diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py index 31dc5acc8995..a4bce73fe0bc 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py @@ -185,6 +185,7 @@ def __init__( scheduler: KarrasDiffusionSchedulers, force_zeros_for_empty_prompt: bool = True, add_watermarker: Optional[bool] = None, + is_cosxl_edit: Optional[bool] = False, ): super().__init__() @@ -201,6 +202,7 @@ def __init__( self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) self.default_sample_size = self.unet.config.sample_size + self.is_cosxl_edit = is_cosxl_edit add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() @@ -551,6 +553,9 @@ def prepare_image_latents( if image_latents.dtype != self.vae.dtype: image_latents = image_latents.to(dtype=self.vae.dtype) + if self.is_cosxl_edit: + image_latents = image_latents * self.vae.config.scaling_factor + return image_latents # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids From e71154acc63364cb65755d7134301a34424e12ee Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 12 Apr 2024 12:53:35 +0530 Subject: [PATCH 2/5] Empty-Commit Co-authored-by: Yiyi Xu From c8f17c2ab45739ebc1a8c8c416a0fd1e9276bfa6 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Fri, 12 Apr 2024 12:54:41 +0530 Subject: [PATCH 3/5] doc --- .../pipeline_stable_diffusion_xl_instruct_pix2pix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py index a4bce73fe0bc..f40333db35ae 100644 --- a/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +++ b/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py @@ -169,6 +169,8 @@ class StableDiffusionXLInstructPix2PixPipeline( Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to watermark output images. If not defined, it will default to True if the package is installed, otherwise no watermarker will be used. + is_cosxl_edit (`bool`, *optional*): + When set the image latents are scaled. """ model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" From 56ad342f359a04213d953fa5a0531f0c470babd4 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Sat, 13 Apr 2024 11:47:39 +0530 Subject: [PATCH 4/5] remove redundant logic. --- src/diffusers/loaders/single_file.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py index 5dc525bf9a15..9182d0afddde 100644 --- a/src/diffusers/loaders/single_file.py +++ b/src/diffusers/loaders/single_file.py @@ -136,7 +136,6 @@ def set_additional_components( original_config, checkpoint=None, model_type=None, - is_cosxl_edit=False, ): components = {} if pipeline_class_name in REFINER_PIPELINES: @@ -148,8 +147,6 @@ def set_additional_components( "force_zeros_for_empty_prompt": False if is_refiner else True, } ) - if pipeline_class_name == "StableDiffusionXLInstructPix2PixPipeline": - components.update({"is_cosxl_edit": is_cosxl_edit}) return components From c68403938918b5934411f3e5975d8efe4ef14e7f Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Sat, 13 Apr 2024 13:02:20 +0530 Subject: [PATCH 5/5] reflect drhuv's comments. --- src/diffusers/loaders/single_file.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/diffusers/loaders/single_file.py b/src/diffusers/loaders/single_file.py index 9182d0afddde..752ef18c7a0b 100644 --- a/src/diffusers/loaders/single_file.py +++ b/src/diffusers/loaders/single_file.py @@ -252,7 +252,6 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): local_files_only = kwargs.pop("local_files_only", False) revision = kwargs.pop("revision", None) torch_dtype = kwargs.pop("torch_dtype", None) - is_cosxl_edit = kwargs.pop("is_cosxl_edit", None) class_name = cls.__name__ @@ -310,7 +309,7 @@ def from_single_file(cls, pretrained_model_link_or_path, **kwargs): init_kwargs.update(components) additional_components = set_additional_components( - class_name, original_config, checkpoint=checkpoint, model_type=model_type, is_cosxl_edit=is_cosxl_edit + class_name, original_config, checkpoint=checkpoint, model_type=model_type ) if additional_components: init_kwargs.update(additional_components)