How to use this VAE on SDXL

[alfredplpl]

Any idea?

from diffusers import DiffusionPipeline,StableDiffusionPipeline
import torch
from consistencydecoder import ConsistencyDecoder
from PIL import Image
import numpy as np

pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16")
pipe = pipe.to("cuda:0")

decoder_consistency = ConsistencyDecoder(device="cuda:0") # Model size: 2.49 GB

prompt = "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k"

latent = pipe(prompt=prompt, output_type="latent")

latent=latent.images[0]
latent=latent.to(torch.float32)/0.13025
latent=latent.unsqueeze(0)
print(latent.size())
with torch.no_grad(), torch.amp.autocast("cuda"):
    consistent_latent = decoder_consistency(latent,schedule=[1.0])

image = consistent_latent[0].cpu().numpy()
image = (image + 1.0) * 127.5
image = image.clip(0, 255).astype(np.uint8)
image = Image.fromarray(image.transpose(1, 2, 0))
image.save("con.png")

[bonlime]

this VAE would not work for SDXL, since it has a different VAE encoder