using ipadapeter_plus and ipadapter_faceid_plus together

[saeedkhanehgir]

Hi,

Thanks for sharing this work. I want to use ipadapeter_plus and ipadapter_faceid_plus together: ipadapeter_plus for changing style and ipadapter_faceid_plus for transferring faces. I tried to write a new StableDiffusionXLPipeline module by merging the modules from pipeline_stable_diffusion_xl_chatglm_256_ipadapter.py and pipeline_stable_diffusion_xl_chatglm_256_ipadapter_FaceID.py script (pipeline.py is my new module, '1.png' is my style image and '2.png' is my face image) pipeline.py.txt

import torch
from transformers import CLIPVisionModelWithProjection,CLIPImageProcessor
from diffusers.utils import load_image
import os,sys
from kolors.models.modeling_chatglm import ChatGLMModel
from kolors.models.tokenization_chatglm import ChatGLMTokenizer

from diffusers import  AutoencoderKL,DPMSolverMultistepScheduler
from kolors.models.unet_2d_condition import UNet2DConditionModel

from diffusers import EulerDiscreteScheduler
from PIL import Image

import cv2
import numpy as np
import insightface
from diffusers.utils import load_image
from insightface.app import FaceAnalysis
from insightface.data import get_image as ins_get_image
from pipeline import StableDiffusionXLPipeline
import os
os.environ["CUDA_VISIBLE_DEVICES"]="3"

class FaceInfoGenerator():
    def __init__(self, root_dir = "./"):
        self.app = FaceAnalysis(name = 'antelopev2', root = root_dir, providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])
        self.app.prepare(ctx_id = 0, det_size = (640, 640))

    def get_faceinfo_one_img(self, image_path):
        face_image = load_image(image_path)
        face_info = self.app.get(cv2.cvtColor(np.array(face_image), cv2.COLOR_RGB2BGR))

        if len(face_info) == 0:
            face_info = None
        else:
            face_info = sorted(face_info, key=lambda x:(x['bbox'][2]-x['bbox'][0])*(x['bbox'][3]-x['bbox'][1]))[-1]  # only use the maximum face
        return face_info

def face_bbox_to_square(bbox):
    ## l, t, r, b to square l, t, r, b
    l,t,r,b = bbox
    cent_x = (l + r) / 2
    cent_y = (t + b) / 2
    w, h = r - l, b - t
    r = max(w, h) / 2

    l0 = cent_x - r
    r0 = cent_x + r
    t0 = cent_y - r
    b0 = cent_y + r

    return [l0, t0, r0, b0]

ckpt_dir = './weights/Kolors'
ipf_model_dir = './weights/Kolors-IP-Adapter-FaceID-Plus'
ip_model_dir = './weights/Kolors-IP-Adapter-Plus'

text_encoder = ChatGLMModel.from_pretrained( f'{ckpt_dir}/text_encoder', torch_dtype = torch.float16).half()
tokenizer = ChatGLMTokenizer.from_pretrained(f'{ckpt_dir}/text_encoder')
vae = AutoencoderKL.from_pretrained(f'{ckpt_dir}/vae', subfolder = "vae", revision = None)
scheduler = EulerDiscreteScheduler.from_pretrained(f'{ckpt_dir}/scheduler')
unet = UNet2DConditionModel.from_pretrained(f'{ckpt_dir}/unet', revision = None).half()

#### clip image encoder for face structure

clip_image_encoder = CLIPVisionModelWithProjection.from_pretrained(f'{ipf_model_dir}/clip-vit-large-patch14-336', ignore_mismatched_sizes=True)
image_encoder = CLIPVisionModelWithProjection.from_pretrained( f'{ip_model_dir}/image_encoder',  ignore_mismatched_sizes=True).to('cuda',dtype=torch.float16)
clip_image_encoder.to("cuda")
clip_image_processor = CLIPImageProcessor(size = 336, crop_size = 336)

pipe = StableDiffusionXLPipeline(
            vae = vae,
            text_encoder = text_encoder,
            tokenizer = tokenizer,
            unet = unet,
            scheduler = scheduler,
            face_clip_encoder = clip_image_encoder,
            face_clip_processor = clip_image_processor,
            force_zeros_for_empty_prompt = False,
            image_encoder=image_encoder,
            feature_extractor=clip_image_processor,
        )
pipe = pipe.to("cuda")
pipe.scheduler = DPMSolverMultistepScheduler.from_config(
            pipe.scheduler.config,use_karras_sigmas=True,

        )

if hasattr(pipe.unet, 'encoder_hid_proj'):
    pipe.unet.text_encoder_hid_proj = pipe.unet.encoder_hid_proj

pipe.load_ip_adapter( f'{ip_model_dir}' , subfolder="", weight_name=["ip_adapter_plus_general.bin"])
pipe.set_ip_adapter_scale([ 0.5 ])

pipe.load_ip_adapter_faceid_plus(f'{ipf_model_dir}/ipa-faceid-plus.bin', device = 'cuda')

ip_adapter_img = Image.open( './comic/1.png' )
pipe.set_face_fidelity_scale(0.8)

face_info_generator = FaceInfoGenerator(root_dir = "./")
img = Image.open('./2.jpg')
face_info = face_info_generator.get_faceinfo_one_img('./2.jpg')

face_bbox_square = face_bbox_to_square(face_info["bbox"])
crop_image = img.crop(face_bbox_square)
crop_image = crop_image.resize((336, 336))
crop_image = [crop_image]

face_embeds = torch.from_numpy(np.array([face_info["embedding"]]))
face_embeds = face_embeds.to('cuda', dtype = torch.float16)

prompt = "a man"
image = pipe(
        prompt = prompt,
        ip_adapter_image=[ ip_adapter_img],
        negative_prompt = "", 
        height = 1024,
        width = 1024,
        num_inference_steps= 25, 
        guidance_scale = 5.0,
        num_images_per_prompt = 1,
        face_crop_image = crop_image,
        face_insightface_embeds = face_embeds,
    ).images[0]

I get below error :

Note that when I load ipadapter_faceid_plus first, everything is ok.

[kentonson]

@saeedkhanehgir Hi, bro, I tried your method, but it seems like the face recognition doesn't work as expected. Are there any updates recently?

[saeedkhanehgir]

@kentonson Hi,I decided not to continue