Open qpc1611094 opened 11 months ago
also erros of: Missing key(s) in state_dict: "1.to_k_ip.weight", "1.to_v_ip.weight", "3.to_k_ip.weight", "3.to_v_ip.weight", "5.to_k_ip.weight", "5.to_v_ip.weight", "7.to_k_ip.weight", "7.to_v_ip.weight", "9.to_k_ip.weight", "9.to_v_ip.weight", "11.to_k_ip.weight", "11.to_v_ip.weight", "13.to_k_ip.weight", "13.to_v_ip.weight", "15.to_k_ip.weight", "15.to_v_ip.weight", "17.to_k_ip.weight", "17.to_v_ip.weight", "19.to_k_ip.weight", "19.to_v_ip.weight", "21.to_k_ip.weight", "21.to_v_ip.weight", "23.to_k_ip.weight", "23.to_v_ip.weight", "25.to_k_ip.weight", "25.to_v_ip.weight", "27.to_k_ip.weight", "27.to_v_ip.weight", "29.to_k_ip.weight", "29.to_v_ip.weight", "31.to_k_ip.weight", "31.to_v_ip.weight".
names_1 = ['down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight']
names_2 = [
"1.to_k_ip.weight", "1.to_v_ip.weight", "3.to_k_ip.weight", "3.to_v_ip.weight", "5.to_k_ip.weight", "5.to_v_ip.weight", "7.to_k_ip.weight", "7.to_v_ip.weight", "9.to_k_ip.weight", "9.to_v_ip.weight", "11.to_k_ip.weight", "11.to_v_ip.weight", "13.to_k_ip.weight", "13.to_v_ip.weight", "15.to_k_ip.weight", "15.to_v_ip.weight", "17.to_k_ip.weight", "17.to_v_ip.weight", "19.to_k_ip.weight", "19.to_v_ip.weight", "21.to_k_ip.weight", "21.to_v_ip.weight", "23.to_k_ip.weight", "23.to_v_ip.weight", "25.to_k_ip.weight", "25.to_v_ip.weight", "27.to_k_ip.weight", "27.to_v_ip.weight", "29.to_k_ip.weight", "29.to_v_ip.weight", "31.to_k_ip.weight", "31.to_v_ip.weight"
]
mapping = {k: v for k, v in zip(names_1, names_2)}
import torch
from safetensors.torch import load_file
ckpt = "output_dir/ffhq/checkpoint-2000/model.safetensors"
sd = load_file(ckpt)
image_proj_sd = {}
ip_sd = {}
for k in sd:
if k.startswith("image_proj_model"):
image_proj_sd[k.replace("image_proj_model.", "")] = sd[k]
elif "_ip." in k:
ip_sd[mapping[k.replace("unet.", "")]] = sd[k]
torch.save({"image_proj": image_proj_sd, "ip_adapter": ip_sd}, "output_dir/ffhq/checkpoint-2000/ip_adapter.bin")
very thanks, there is still a little bug: _RuntimeError: Error(s) in loading state_dict for ModuleList: size mismatch for 19.to_k_ip.weight: copying a param with shape torch.Size([1280, 768]) from checkpoint, the shape in current model is torch.Size([640, 768]). size mismatch for 19.to_v_ip.weight: copying a param with shape torch.Size([1280, 768]) from checkpoint, the shape in current model is torch.Size([640, 768]). size mismatch for 25.to_k_ip.weight: copying a param with shape torch.Size([640, 768]) from checkpoint, the shape in current model is torch.Size([320, 768]). size mismatch for 25.to_v_ip.weight: copying a param with shape torch.Size([640, 768]) from checkpoint, the shape in current model is torch.Size([320, 768]). size mismatch for 31.to_k_ip.weight: copying a param with shape torch.Size([320, 768]) from checkpoint, the shape in current model is torch.Size([1280, 768]). size mismatch for 31.to_vip.weight: copying a param with shape torch.Size([320, 768]) from checkpoint, the shape in current model is torch.Size([1280, 768]).
names_1 = [ "down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight", "up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight", "up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight" ] names_ 2 = [ "1.to_k_ip.weight", "1.to_v_ip.weight", "3.to_k_ip.weight", "3.to_v_ip.weight", "5.to_k_ip.weight", "5.to_v_ip.weight", "7.to_k_ip.weight", "7.to_v_ip.weight", "9.to_k_ip.weight", "9.to_v_ip.weight", "11.to_k_ip.weight", "11.to_v_ip.weight", "13.to_k_ip.weight", "13.to_v_ip.weight", "15.to_k_ip.weight", "15.to_v_ip.weight", "17.to_k_ip.weight", "17.to_v_ip.weight", "19.to_k_ip.weight", "19.to_v_ip.weight", "21.to_k_ip.weight", "21.to_v_ip.weight", "23.to_k_ip.weight", "23.to_v_ip.weight", "25.to_k_ip.weight", "25.to_v_ip.weight", "27.to_k_ip.weight", "27.to_v_ip.weight", "29.to_k_ip.weight", "29.to_v_ip.weight", "31.to_k_ip.weight", "31.to_v_ip.weight" ] mapping = {k: v for k, v in zip(names_1, names_2)} import torch from safetensors.torch import load_file ckpt = "output_dir/ffhq/checkpoint-2000/model.safetensors" sd = load_file(ckpt) image_proj_sd = {} ip_sd = {} for k in sd: if k.startswith("image_proj_model"): image_proj_sd[k.replace("image_proj_model.", "")] = sd[k] elif "_ip." in k: ip_sd[mapping[k.replace("unet.", "")]] = sd[k] torch.save({"image_proj": image_proj_sd, "ip_adapter": ip_sd}, "output_dir/ffhq/checkpoint-2000/ip_adapter.bin")
I'm having the same problem as you.
names_1 = ['down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.0.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.0.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'down_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.1.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.2.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.1.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'up_blocks.3.attentions.2.transformer_blocks.0.attn2.processor.to_v_ip.weight', 'mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_k_ip.weight', 'mid_block.attentions.0.transformer_blocks.0.attn2.processor.to_v_ip.weight'] names_2 = [ "1.to_k_ip.weight", "1.to_v_ip.weight", "3.to_k_ip.weight", "3.to_v_ip.weight", "5.to_k_ip.weight", "5.to_v_ip.weight", "7.to_k_ip.weight", "7.to_v_ip.weight", "9.to_k_ip.weight", "9.to_v_ip.weight", "11.to_k_ip.weight", "11.to_v_ip.weight", "13.to_k_ip.weight", "13.to_v_ip.weight", "15.to_k_ip.weight", "15.to_v_ip.weight", "17.to_k_ip.weight", "17.to_v_ip.weight", "19.to_k_ip.weight", "19.to_v_ip.weight", "21.to_k_ip.weight", "21.to_v_ip.weight", "23.to_k_ip.weight", "23.to_v_ip.weight", "25.to_k_ip.weight", "25.to_v_ip.weight", "27.to_k_ip.weight", "27.to_v_ip.weight", "29.to_k_ip.weight", "29.to_v_ip.weight", "31.to_k_ip.weight", "31.to_v_ip.weight" ] mapping = {k: v for k, v in zip(names_1, names_2)} import torch from safetensors.torch import load_file ckpt = "output_dir/ffhq/checkpoint-2000/model.safetensors" sd = load_file(ckpt) image_proj_sd = {} ip_sd = {} for k in sd: if k.startswith("image_proj_model"): image_proj_sd[k.replace("image_proj_model.", "")] = sd[k] elif "_ip." in k: ip_sd[mapping[k.replace("unet.", "")]] = sd[k] torch.save({"image_proj": image_proj_sd, "ip_adapter": ip_sd}, "output_dir/ffhq/checkpoint-2000/ip_adapter.bin")
I updated the script, can you try again?
it works, thanks
it works, thanks so much
Have your solved this?
I have the same issue with you. Have you solved this?
Hi, I have trained a new model but meet errors when testing, I did it as:
train a model with:
the image_encoder is download from your provided link, and this can work
convert weight with:
torch.save({"image_proj": image_proj_sd, "ip_adapter": ip_sd}, "output_dir/ffhq/checkpoint-2000/ip_adapter.bin")
base_model_path = "stable-diffusion-v1-5" vae_model_path = "sd-vae-ft-mse" image_encoder_path = "image_encoder" ip_ckpt = "output_dir/ffhq/checkpoint-2000/ip_adapter.bin" device = "cuda"
def image_grid(imgs, rows, cols): assert len(imgs) == rows*cols
noise_scheduler = DDIMScheduler( num_train_timesteps=1000, beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", clip_sample=False, set_alpha_to_one=False, steps_offset=1, ) vae = AutoencoderKL.from_pretrained(vae_model_path).to(dtype=torch.float16)
pipe = StableDiffusionPipeline.from_pretrained( base_model_path, torch_dtype=torch.float16, scheduler=noise_scheduler, vae=vae, feature_extractor=None, safety_checker=None )
ip_model = IPAdapter(pipe, image_encoder_path, ip_ckpt, device)