I can't load loras

[loboere]

I try to load loras with pipeline.pipe.load_lora_weights("/kaggle/input/lorass/acuarelac1400.safetensors")

I don't know if it is the correct way, it would be helpful if you told me how to load loras

but i get

---------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[175], line 23
     21 seed_everything(seed)
     22 #out=pipeline.inference(prompt, init_image,  mask_image , 0.8, (1, H, W), neg_prompt, id_embeddings, id_scale, scale, steps  )
---> 23 out=pipeline.pipe(prompt=prompt,
     24   image=init_image,
     25   mask_image=mask_image,
     26   strength=0.8,
     27   negative_prompt=neg_prompt,
     28   num_images_per_prompt=1,
     29   height=H,
     30   width=W,
     31   num_inference_steps=steps,
     32   guidance_scale= scale,
     33   cross_attention_kwargs={ 'id_embedding': id_embeddings, 'id_scale': id_scale},)
     35 out[0]

File /opt/conda/lib/python3.10/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File /opt/conda/lib/python3.10/site-packages/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py:1707, in StableDiffusionXLInpaintPipeline.__call__(self, prompt, prompt_2, image, mask_image, masked_image_latents, height, width, strength, num_inference_steps, timesteps, denoising_start, denoising_end, guidance_scale, negative_prompt, negative_prompt_2, num_images_per_prompt, eta, generator, latents, prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, ip_adapter_image, output_type, return_dict, cross_attention_kwargs, guidance_rescale, original_size, crops_coords_top_left, target_size, negative_original_size, negative_crops_coords_top_left, negative_target_size, aesthetic_score, negative_aesthetic_score, clip_skip, callback_on_step_end, callback_on_step_end_tensor_inputs, **kwargs)
   1705 if ip_adapter_image is not None:
   1706     added_cond_kwargs["image_embeds"] = image_embeds
-> 1707 noise_pred = self.unet(
   1708     latent_model_input,
   1709     t,
   1710     encoder_hidden_states=prompt_embeds,
   1711     timestep_cond=timestep_cond,
   1712     cross_attention_kwargs=self.cross_attention_kwargs,
   1713     added_cond_kwargs=added_cond_kwargs,
   1714     return_dict=False,
   1715 )[0]
   1717 # perform guidance
   1718 if self.do_classifier_free_guidance:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/diffusers/models/unet_2d_condition.py:1112, in UNet2DConditionModel.forward(self, sample, timestep, encoder_hidden_states, class_labels, timestep_cond, attention_mask, cross_attention_kwargs, added_cond_kwargs, down_block_additional_residuals, mid_block_additional_residual, down_intrablock_additional_residuals, encoder_attention_mask, return_dict)
   1109     if is_adapter and len(down_intrablock_additional_residuals) > 0:
   1110         additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0)
-> 1112     sample, res_samples = downsample_block(
   1113         hidden_states=sample,
   1114         temb=emb,
   1115         encoder_hidden_states=encoder_hidden_states,
   1116         attention_mask=attention_mask,
   1117         cross_attention_kwargs=cross_attention_kwargs,
   1118         encoder_attention_mask=encoder_attention_mask,
   1119         **additional_residuals,
   1120     )
   1121 else:
   1122     sample, res_samples = downsample_block(hidden_states=sample, temb=emb, scale=lora_scale)

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/diffusers/models/unet_2d_blocks.py:1160, in CrossAttnDownBlock2D.forward(self, hidden_states, temb, encoder_hidden_states, attention_mask, cross_attention_kwargs, encoder_attention_mask, additional_residuals)
   1158 else:
   1159     hidden_states = resnet(hidden_states, temb, scale=lora_scale)
-> 1160     hidden_states = attn(
   1161         hidden_states,
   1162         encoder_hidden_states=encoder_hidden_states,
   1163         cross_attention_kwargs=cross_attention_kwargs,
   1164         attention_mask=attention_mask,
   1165         encoder_attention_mask=encoder_attention_mask,
   1166         return_dict=False,
   1167     )[0]
   1169 # apply additional residuals to the output of the last pair of resnet and attention blocks
   1170 if i == len(blocks) - 1 and additional_residuals is not None:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/diffusers/models/transformer_2d.py:392, in Transformer2DModel.forward(self, hidden_states, encoder_hidden_states, timestep, added_cond_kwargs, class_labels, cross_attention_kwargs, attention_mask, encoder_attention_mask, return_dict)
    380         hidden_states = torch.utils.checkpoint.checkpoint(
    381             create_custom_forward(block),
    382             hidden_states,
   (...)
    389             **ckpt_kwargs,
    390         )
    391     else:
--> 392         hidden_states = block(
    393             hidden_states,
    394             attention_mask=attention_mask,
    395             encoder_hidden_states=encoder_hidden_states,
    396             encoder_attention_mask=encoder_attention_mask,
    397             timestep=timestep,
    398             cross_attention_kwargs=cross_attention_kwargs,
    399             class_labels=class_labels,
    400         )
    402 # 3. Output
    403 if self.is_input_continuous:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/diffusers/models/attention.py:366, in BasicTransformerBlock.forward(self, hidden_states, attention_mask, encoder_hidden_states, encoder_attention_mask, timestep, cross_attention_kwargs, class_labels, added_cond_kwargs)
    363     if self.pos_embed is not None and self.use_ada_layer_norm_single is False:
    364         norm_hidden_states = self.pos_embed(norm_hidden_states)
--> 366     attn_output = self.attn2(
    367         norm_hidden_states,
    368         encoder_hidden_states=encoder_hidden_states,
    369         attention_mask=encoder_attention_mask,
    370         **cross_attention_kwargs,
    371     )
    372     hidden_states = attn_output + hidden_states
    374 # 4. Feed-forward

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/diffusers/models/attention_processor.py:527, in Attention.forward(self, hidden_states, encoder_hidden_states, attention_mask, **cross_attention_kwargs)
    508 r"""
    509 The forward method of the `Attention` class.
    510 
   (...)
    522     `torch.Tensor`: The output of the attention layer.
    523 """
    524 # The `Attention` class can call different attention processors / attention functions
    525 # here we simply pass along all tensors to the selected processor class
    526 # For standard processors that are defined here, `**cross_attention_kwargs` is empty
--> 527 return self.processor(
    528     self,
    529     hidden_states,
    530     encoder_hidden_states=encoder_hidden_states,
    531     attention_mask=attention_mask,
    532     **cross_attention_kwargs,
    533 )

File /kaggle/working/PuLID/pulid/attention_processor.py:365, in IDAttnProcessor2_0.__call__(self, attn, hidden_states, encoder_hidden_states, attention_mask, temb, id_embedding, id_scale)
    359 else:
    360     zero_tensor = torch.zeros(
    361         (id_embedding.size(0), NUM_ZERO, id_embedding.size(-1)),
    362         dtype=id_embedding.dtype,
    363         device=id_embedding.device,
    364     )
--> 365     id_key = self.id_to_k(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype)
    366     id_value = self.id_to_v(torch.cat((id_embedding, zero_tensor), dim=1)).to(query.dtype)
    368 id_key = id_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/linear.py:114, in Linear.forward(self, input)
    113 def forward(self, input: Tensor) -> Tensor:
--> 114     return F.linear(input, self.weight, self.bias)

RuntimeError: expected scalar type Float but found Half

I don't know if it is the correct way, it would be helpful if you told me how to load loras

[mmlingyu]

expected mat1 and mat2 to have the same dtype, but got: float!= c10::Half ， the same problem. any one is fix？