From d33e1bf5636c55f5aea43f02ec323f16cb57a488 Mon Sep 17 00:00:00 2001 From: Sean McLellan Date: Wed, 24 Aug 2022 12:02:36 -0400 Subject: [PATCH 1/4] Add simple way to make variants --- ldm/simplet2i.py | 5 +++-- scripts/dream.py | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index c7f6263816d..8ffeec8dcb2 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -143,7 +143,8 @@ def __init__(self, def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, - cfg_scale=None,ddim_eta=None,strength=None,init_img=None,skip_normalize=False): + cfg_scale=None,ddim_eta=None,strength=None,init_img=None, + skip_normalize=False,variants=None): """ Generate an image from the prompt, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] @@ -268,7 +269,7 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, # There is lots of shared code between this and txt2img and should be refactored. def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, - cfg_scale=None,ddim_eta=None,strength=None,skip_normalize=False): + cfg_scale=None,ddim_eta=None,strength=None,skip_normalize=False,variants=None): """ Generate an image from the prompt and the initial image, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] diff --git a/scripts/dream.py b/scripts/dream.py index fb8fec2384a..17b9b6c389e 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -4,6 +4,7 @@ import atexit import os import sys +import copy from PIL import Image,PngImagePlugin # readline unavailable on windows systems @@ -162,8 +163,29 @@ def main_loop(t2i,parser,log,infile): results = t2i.txt2img(**vars(opt)) else: results = t2i.img2img(**vars(opt)) + + allVariantResults = [] + if opt.variants is not None: + print(f"Generating {opt.variants} variant(s)...") + newopt = copy.deepcopy(opt) + newopt.variants = None + for r in results: + resultPath = r[0] + print(f"\t generating variant for {resultPath}") + for j in range(0, opt.variants): + newopt.init_img = resultPath + print(f"{newopt.init_img}") + variantResults = t2i.img2img(**vars(newopt)) + allVariantResults.append([newopt,variantResults]) + print(f"{opt.variants} Variants generated!") + print("Outputs:") write_log_message(t2i,opt,results,log) + + if len(allVariantResults)>0: + print("Variant outputs:") + for vr in allVariantResults: + write_log_message(t2i,vr[0],vr[1],log) print("goodbye!") @@ -285,6 +307,7 @@ def create_cmd_parser(): parser.add_argument('-i','--individual',action='store_true',help="generate individual files (default)") parser.add_argument('-I','--init_img',type=str,help="path to input image (supersedes width and height)") parser.add_argument('-f','--strength',default=0.75,type=float,help="strength for noising/unnoising. 0.0 preserves image exactly, 1.0 replaces it completely") + parser.add_argument('-v','--variants',type=int,help="number of variants to generate of each image") parser.add_argument('-x','--skip_normalize',action='store_true',help="skip subprompt weight normalization") return parser @@ -293,7 +316,7 @@ def setup_readline(): readline.set_completer(Completer(['cd','pwd', '--steps','-s','--seed','-S','--iterations','-n','--batch_size','-b', '--width','-W','--height','-H','--cfg_scale','-C','--grid','-g', - '--individual','-i','--init_img','-I','--strength','-f']).complete) + '--individual','-i','--init_img','-I','--strength','-f','-v','--variants']).complete) readline.set_completer_delims(" ") readline.parse_and_bind('tab: complete') load_history() From b5cdbd3b0b7ee6d7cadb8ba4fbc5cdce2820c46d Mon Sep 17 00:00:00 2001 From: Sean McLellan Date: Wed, 24 Aug 2022 13:14:08 -0400 Subject: [PATCH 2/4] Fixes issue with cuda/current mismatch --- ldm/models/diffusion/ddim.py | 6 +++++- ldm/models/diffusion/plms.py | 7 ++++++- ldm/simplet2i.py | 14 +++++++------- scripts/dream.py | 4 ++-- src/k-diffusion | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ldm/models/diffusion/ddim.py b/ldm/models/diffusion/ddim.py index 2ebaeabd22b..ddf786b5a85 100644 --- a/ldm/models/diffusion/ddim.py +++ b/ldm/models/diffusion/ddim.py @@ -10,13 +10,17 @@ class DDIMSampler(object): - def __init__(self, model, schedule="linear", **kwargs): + def __init__(self, model, schedule="linear", device="cuda", **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule + self.device = device def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device(self.device): + attr = attr.to(torch.device(self.device)) setattr(self, name, attr) def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True): diff --git a/ldm/models/diffusion/plms.py b/ldm/models/diffusion/plms.py index 5d09f023f3b..5eafe1d7ce5 100644 --- a/ldm/models/diffusion/plms.py +++ b/ldm/models/diffusion/plms.py @@ -9,13 +9,18 @@ class PLMSSampler(object): - def __init__(self, model, schedule="linear", **kwargs): + def __init__(self, model, schedule="linear", device="cuda", **kwargs): super().__init__() self.model = model self.ddpm_num_timesteps = model.num_timesteps self.schedule = schedule + self.device = device def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device(self.device): + attr = attr.to(torch.device(self.device)) + setattr(self, name, attr) def make_schedule(self, ddim_num_steps, ddim_discretize="uniform", ddim_eta=0., verbose=True): diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 183e2448eb6..09141c8ebb0 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -58,7 +58,6 @@ import os from omegaconf import OmegaConf from PIL import Image -import PIL from tqdm import tqdm, trange from itertools import islice from einops import rearrange, repeat @@ -286,7 +285,8 @@ def txt2img(self,prompt,outdir=None,batch_size=None,iterations=None, @torch.no_grad() def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=None, steps=None,seed=None,grid=None,individual=None,width=None,height=None, - cfg_scale=None,ddim_eta=None,strength=None,skip_normalize=False): + cfg_scale=None,ddim_eta=None,strength=None,embedding_path=None, + skip_normalize=False,variants=None): """ Generate an image from the prompt and the initial image, writing iteration images into the outdir The output is a list of lists in the format: [[filename1,seed1], [filename2,seed2],...] @@ -324,7 +324,7 @@ def img2img(self,prompt,outdir=None,init_img=None,batch_size=None,iterations=Non # PLMS sampler not supported yet, so ignore previous sampler if self.sampler_name!='ddim': print(f"sampler '{self.sampler_name}' is not yet supported. Using DDM sampler") - sampler = DDIMSampler(model) + sampler = DDIMSampler(model, device=self.device) else: sampler = self.sampler @@ -461,9 +461,9 @@ def load_model(self): msg = f'setting sampler to {self.sampler_name}' if self.sampler_name=='plms': - self.sampler = PLMSSampler(self.model) + self.sampler = PLMSSampler(self.model, device=self.device) elif self.sampler_name == 'ddim': - self.sampler = DDIMSampler(self.model) + self.sampler = DDIMSampler(self.model, device=self.device) elif self.sampler_name == 'k_dpm_2_a': self.sampler = KSampler(self.model,'dpm_2_ancestral') elif self.sampler_name == 'k_dpm_2': @@ -478,7 +478,7 @@ def load_model(self): self.sampler = KSampler(self.model,'lms') else: msg = f'unsupported sampler {self.sampler_name}, defaulting to plms' - self.sampler = PLMSSampler(self.model) + self.sampler = PLMSSampler(self.model, device=self.device) print(msg) @@ -505,7 +505,7 @@ def _load_img(self,path): w, h = image.size print(f"loaded input image of size ({w}, {h}) from {path}") w, h = map(lambda x: x - x % 32, (w, h)) # resize to integer multiple of 32 - image = image.resize((w, h), resample=PIL.Image.LANCZOS) + image = image.resize((w, h), resample=Image.Resampling.LANCZOS) image = np.array(image).astype(np.float32) / 255.0 image = image[None].transpose(0, 3, 1, 2) image = torch.from_numpy(image) diff --git a/scripts/dream.py b/scripts/dream.py index 94d5e9a1a6a..886d8bd6822 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -191,16 +191,16 @@ def main_loop(t2i,parser,log,infile): print(f"{newopt.init_img}") try: variantResults = t2i.img2img(**vars(newopt)) + allVariantResults.append([newopt,variantResults]) except AssertionError as e: print(e) continue - allVariantResults.append([newopt,variantResults]) print(f"{opt.variants} Variants generated!") print("Outputs:") write_log_message(t2i,opt,results,log) - if len(allVariantResults)>0: + if allVariantResults: print("Variant outputs:") for vr in allVariantResults: write_log_message(t2i,vr[0],vr[1],log) diff --git a/src/k-diffusion b/src/k-diffusion index db579906874..ef1bf07627c 160000 --- a/src/k-diffusion +++ b/src/k-diffusion @@ -1 +1 @@ -Subproject commit db5799068749bf3a6d5845120ed32df16b7d883b +Subproject commit ef1bf07627c9a10ba9137e68a0206b844544a7d9 From ca82acfd3b2a74e37891f627cff9102795fa600f Mon Sep 17 00:00:00 2001 From: Sean McLellan Date: Wed, 24 Aug 2022 13:33:19 -0400 Subject: [PATCH 3/4] Remove unnecessary print, small optmi --- scripts/dream.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/dream.py b/scripts/dream.py index dca26b16a0c..37d213155a2 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -183,11 +183,9 @@ def main_loop(t2i,parser,log,infile): newopt = copy.deepcopy(opt) newopt.variants = None for r in results: - resultPath = r[0] + newopt.init_img = resultPath = r[0] print(f"\t generating variant for {resultPath}") for j in range(0, opt.variants): - newopt.init_img = resultPath - print(f"{newopt.init_img}") try: variantResults = t2i.img2img(**vars(newopt)) allVariantResults.append([newopt,variantResults]) From ee10021ea21dfe611a273a0b69c01b1c80d15567 Mon Sep 17 00:00:00 2001 From: Sean McLellan Date: Wed, 24 Aug 2022 13:36:27 -0400 Subject: [PATCH 4/4] bikeshedding --- scripts/dream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dream.py b/scripts/dream.py index 37d213155a2..211813c4c2e 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -183,8 +183,8 @@ def main_loop(t2i,parser,log,infile): newopt = copy.deepcopy(opt) newopt.variants = None for r in results: - newopt.init_img = resultPath = r[0] - print(f"\t generating variant for {resultPath}") + newopt.init_img = r[0] + print(f"\t generating variant for {newopt.init_img}") for j in range(0, opt.variants): try: variantResults = t2i.img2img(**vars(newopt))