diff --git a/configs/animatediff/README.md b/configs/animatediff/README.md index 1432999fe..a1337427a 100644 --- a/configs/animatediff/README.md +++ b/configs/animatediff/README.md @@ -35,6 +35,12 @@ This model has several weights including vae, unet and clip. You should download Running the following codes, you can get a text-generated image. +### Reccomendation + +It's highly recommended to install [xformers](https://github.com/facebookresearch/xformers). It would save about 20G memory for 512\*512 resolution generation. + +### Steps + 1. Download [ToonYou](https://civitai.com/api/download/models/78775) and MotionModule checkpoint ```bash @@ -52,11 +58,11 @@ wget https://civitai.com/api/download/models/78775 -P DreamBooth_LoRA/ --content ```python models_path = {Your Checkpoints Path} motion_module_cfg=dict( - path={Your MotionModule path} + path={Your MotionModule Path} ), dream_booth_lora_cfg=dict( type='ToonYou', - path={Your Dreambooth_Lora path}, + path={Your Dreambooth_Lora Path}, steps=25, guidance_scale=7.5) ``` @@ -106,7 +112,7 @@ time_str = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") savedir = f"samples/{Path(cfg.model['dream_booth_lora_cfg']['type']).stem}-{time_str}" os.makedirs(savedir) for prompt_idx, (prompt, n_prompt, random_seed) in enumerate(zip(prompts, negative_prompts, random_seeds)): - output_dict = animatediff.infer(prompt,negative_prompt=n_prompt, video_length=16, height=512, width=512, seed=random_seed,num_inference_steps=cfg.model['dream_booth_lora_cfg']['steps']) + output_dict = animatediff.infer(prompt,negative_prompt=n_prompt, video_length=16, height=256, width=256, seed=random_seed,num_inference_steps=cfg.model['dream_booth_lora_cfg']['steps']) sample = output_dict['samples'] prompt = "-".join((prompt.replace("/", "").split(" ")[:10])) save_videos_grid(sample, f"{savedir}/sample/{sample_idx}-{prompt}.gif") diff --git a/configs/animatediff/README_zh-CN.md b/configs/animatediff/README_zh-CN.md index 239e52dd3..9d2165e75 100644 --- a/configs/animatediff/README_zh-CN.md +++ b/configs/animatediff/README_zh-CN.md @@ -33,10 +33,14 @@ | [RealisticVision](./animatediff_RealisticVision.py) | [model](https://civitai.com/api/download/models/29460) | | [RealisticVision_v2](./animatediff_RealisticVision_v2.py) | [model](https://huggingface.co/guoyww/animatediff/resolve/main/mm_sd_v15_v2.ckpt) | -## Quick Start +## 快速开始 运行以下代码,你可以使用AnimateDiff通过文本生成视频。 +#### 小建议 + +强烈推荐安装 [xformers](https://github.com/facebookresearch/xformers),512\*512分辨率可以节省约20G显存。 + 1. 下载 [ToonYou](https://civitai.com/api/download/models/78775) 和 MotionModule 权重 ```bash @@ -54,11 +58,11 @@ wget https://civitai.com/api/download/models/78775 -P DreamBooth_LoRA/ --content ```python models_path = {Your Checkpoints Path} motion_module_cfg=dict( - path={Your MotionModule path} + path={Your MotionModule Path} ), dream_booth_lora_cfg=dict( type='ToonYou', - path={Your Dreambooth_Lora path}, + path={Your Dreambooth_Lora Path}, steps=25, guidance_scale=7.5) ``` @@ -107,7 +111,7 @@ time_str = datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S") savedir = f"samples/{Path(cfg.model['dream_booth_lora_cfg']['type']).stem}-{time_str}" os.makedirs(savedir) for prompt_idx, (prompt, n_prompt, random_seed) in enumerate(zip(prompts, negative_prompts, random_seeds)): - output_dict = animatediff.infer(prompt,negative_prompt=n_prompt, video_length=16, height=512, width=512, seed=random_seed,num_inference_steps=cfg.model['dream_booth_lora_cfg']['steps']) + output_dict = animatediff.infer(prompt,negative_prompt=n_prompt, video_length=16, height=256, width=256, seed=random_seed,num_inference_steps=cfg.model['dream_booth_lora_cfg']['steps']) sample = output_dict['samples'] prompt = "-".join((prompt.replace("/", "").split(" ")[:10])) save_videos_grid(sample, f"{savedir}/sample/{sample_idx}-{prompt}.gif") @@ -189,7 +193,7 @@ prompt: ``` -## Citation +## 引用 ```bibtex @article{guo2023animatediff, diff --git a/mmagic/models/editors/animatediff/animatediff_utils.py b/mmagic/models/editors/animatediff/animatediff_utils.py index 1d513d284..44d8923f8 100644 --- a/mmagic/models/editors/animatediff/animatediff_utils.py +++ b/mmagic/models/editors/animatediff/animatediff_utils.py @@ -1096,4 +1096,4 @@ def save_videos_grid(videos: torch.Tensor, outputs.append(x) os.makedirs(os.path.dirname(path), exist_ok=True) - imageio.mimsave(path, outputs, fps=fps) + imageio.mimsave(path, outputs, duration=1000 * 1 / fps) diff --git a/mmagic/models/editors/animatediff/attention_3d.py b/mmagic/models/editors/animatediff/attention_3d.py index 58fa2f7b0..de8464ddd 100644 --- a/mmagic/models/editors/animatediff/attention_3d.py +++ b/mmagic/models/editors/animatediff/attention_3d.py @@ -467,7 +467,10 @@ def __init__( # You can set slice_size with `set_attention_slice` self.sliceable_head_dim = heads self._slice_size = None - self._use_memory_efficient_attention_xformers = True + if xformers is not None: + self._use_memory_efficient_attention_xformers = True + else: + self._use_memory_efficient_attention_xformers = False self.added_kv_proj_dim = added_kv_proj_dim if norm_num_groups is not None: