dosubot[bot]
commented
2 months ago Hey, @marc2608! I'm here to assist you with any bugs, questions, or contributions. Let's work together to solve the issue at hand.
I found a similar closed issue: [Bug]: Can't use "ad_model": "mediapipe_face_all" which was last updated on June 30, 2023. The issue was due to a typo, and the correct model type is "mediapipe_face_full" [1].
The connection errors you're experiencing with the mediapipe models in adetailer are likely due to the mediapipe library not being imported at the top of the script. The mediapipe library is imported within the functions, which might be causing issues if the imports fail or if there are environment-specific problems.
To resolve this, ensure that the mediapipe library is correctly installed and import it at the top of the script. Here is the modified code with the mediapipe import at the top:
from __future__ import annotations
from functools import partial
import cv2
import numpy as np
from PIL import Image, ImageDraw
import mediapipe as mp # Import mediapipe at the top
from adetailer import PredictOutput
from adetailer.common import create_bbox_from_mask, create_mask_from_bbox
def mediapipe_predict(
model_type: str, image: Image.Image, confidence: float = 0.3
) -> PredictOutput:
mapping = {
"mediapipe_face_short": partial(mediapipe_face_detection, 0),
"mediapipe_face_full": partial(mediapipe_face_detection, 1),
"mediapipe_face_mesh": mediapipe_face_mesh,
"mediapipe_face_mesh_eyes_only": mediapipe_face_mesh_eyes_only,
}
if model_type in mapping:
func = mapping[model_type]
try:
return func(image, confidence)
except Exception:
return PredictOutput()
msg = f"[-] ADetailer: Invalid mediapipe model type: {model_type}, Available: {list(mapping.keys())!r}"
raise RuntimeError(msg)
def mediapipe_face_detection(
model_type: int, image: Image.Image, confidence: float = 0.3
) -> PredictOutput[float]:
img_width, img_height = image.size
mp_face_detection = mp.solutions.face_detection
draw_util = mp.solutions.drawing_utils
img_array = np.array(image)
with mp_face_detection.FaceDetection(
model_selection=model_type, min_detection_confidence=confidence
) as face_detector:
pred = face_detector.process(img_array)
if pred.detections is None:
return PredictOutput()
preview_array = img_array.copy()
bboxes = []
for detection in pred.detections:
draw_util.draw_detection(preview_array, detection)
bbox = detection.location_data.relative_bounding_box
x1 = bbox.xmin * img_width
y1 = bbox.ymin * img_height
w = bbox.width * img_width
h = bbox.height * img_height
x2 = x1 + w
y2 = y1 + h
bboxes.append([x1, y1, x2, y2])
masks = create_mask_from_bbox(bboxes, image.size)
preview = Image.fromarray(preview_array)
return PredictOutput(bboxes=bboxes, masks=masks, preview=preview)
def mediapipe_face_mesh(
image: Image.Image, confidence: float = 0.3
) -> PredictOutput[int]:
mp_face_mesh = mp.solutions.face_mesh
draw_util = mp.solutions.drawing_utils
drawing_styles = mp.solutions.drawing_styles
w, h = image.size
with mp_face_mesh.FaceMesh(
static_image_mode=True, max_num_faces=20, min_detection_confidence=confidence
) as face_mesh:
arr = np.array(image)
pred = face_mesh.process(arr)
if pred.multi_face_landmarks is None:
return PredictOutput()
preview = arr.copy()
masks = []
for landmarks in pred.multi_face_landmarks:
draw_util.draw_landmarks(
image=preview,
landmark_list=landmarks,
connections=mp_face_mesh.FACEMESH_TESSELATION,
landmark_drawing_spec=None,
connection_drawing_spec=drawing_styles.get_default_face_mesh_tesselation_style(),
)
points = np.array(
[[land.x * w, land.y * h] for land in landmarks.landmark], dtype=int
)
outline = cv2.convexHull(points).reshape(-1).tolist()
mask = Image.new("L", image.size, "black")
draw = ImageDraw.Draw(mask)
draw.polygon(outline, fill="white")
masks.append(mask)
bboxes = create_bbox_from_mask(masks, image.size)
preview = Image.fromarray(preview)
return PredictOutput(bboxes=bboxes, masks=masks, preview=preview)
def mediapipe_face_mesh_eyes_only(
image: Image.Image, confidence: float = 0.3
) -> PredictOutput[int]:
mp_face_mesh = mp.solutions.face_mesh
left_idx = np.array(list(mp_face_mesh.FACEMESH_LEFT_EYE)).flatten()
right_idx = np.array(list(mp_face_mesh.FACEMESH_RIGHT_EYE)).flatten()
w, h = image.size
with mp_face_mesh.FaceMesh(
static_image_mode=True, max_num_faces=20, min_detection_confidence=confidence
) as face_mesh:
arr = np.array(image)
pred = face_mesh.process(arr)
if pred.multi_face_landmarks is None:
return PredictOutput()
preview = image.copy()
masks = []
for landmarks in pred.multi_face_landmarks:
points = np.array(
[[land.x * w, land.y * h] for land in landmarks.landmark], dtype=int
)
left_eyes = points[left_idx]
right_eyes = points[right_idx]
left_outline = cv2.convexHull(left_eyes).reshape(-1).tolist()
right_outline = cv2.convexHull(right_eyes).reshape(-1).tolist()
mask = Image.new("L", image.size, "black")
draw = ImageDraw.Draw(mask)
for outline in (left_outline, right_outline):
draw.polygon(outline, fill="white")
masks.append(mask)
bboxes = create_bbox_from_mask(masks, image.size)
preview = draw_preview(preview, bboxes, masks)
return PredictOutput(bboxes=bboxes, masks=masks, preview=preview)
def draw_preview(
preview: Image.Image, bboxes: list[list[int]], masks: list[Image.Image]
) -> Image.Image:
red = Image.new("RGB", preview.size, "red")
for mask in masks:
masked = Image.composite(red, preview, mask)
preview = Image.blend(preview, masked, 0.25)
draw = ImageDraw.Draw(preview)
for bbox in bboxes:
draw.rectangle(bbox, outline="red", width=2)
return previewEnsure that the mediapipe library is installed in your environment using:
pip install mediapipeAdditionally, you can use the provided test script to verify that the mediapipe models are working correctly:
import pytest
from PIL import Image
from adetailer.mediapipe import mediapipe_predict
@pytest.mark.parametrize(
"model_name",
[
"mediapipe_face_short",
"mediapipe_face_full",
"mediapipe_face_mesh",
"mediapipe_face_mesh_eyes_only",
],
)
def test_mediapipe(sample_image2: Image.Image, model_name: str):
result = mediapipe_predict(model_name, sample_image2)
assert result.preview is not NoneThis should help resolve the connection errors related to the mediapipe models in adetailer [2][3].
To continue talking to Dosu, mention @dosu.
marc2608
Describe the bug
Hi, all the models in adetailer work fine except for the 4 mediapipes (mediapipe_face_full; mediapipe_face_short; mediapipe_face_mesh; mediapipe_facemesh_eyes_only). None of them want to work. For example, in text-to-image the image starts to be generated, gets to 100%, then we see that the model doesn't perform a detection, then it stops and generates several error inserts on the top right of the webui A1111. Any ideas on what might go wrong?
Steps to reproduce
installation: version: [v1.10.1] python: 3.10.6 • torch: 2.1.2+cu121 • xformers: 0.0.23.post1 • gradio: 3.41.2 -generating an image - SDE kARRAS - using A DETAILER with a mediapipe model.
Screenshots
Console logs, from start to end.
List of installed extensions
LDSR
Lora
ScuNET
SwinIR
canvas-zoom-and-pan
extra-options-section
hypertile
mobile
postprocessing-for-training
prompt-bracket-checker
soft-inpainting
--sd-webui-ar-plus
--sd-webui-ar-plusplus
ABG_extension
CharacteristicGuidanceWebUI
DreamArtist-sd-webui-extension FreeU
IF_prompt_MKR
OneButtonPrompt
PBRemTools
Stable-Diffusion-Webui-Civitai-Helper
StylePile
Stylez
a-person-mask-generator adetailer
canvas-zoom
clip-interrogator-ext custom-hires-fix-for-automatic1111
loopback_scaler
model-keyword
multidiffusion-upscaler-for-automatic1111 next-diffusion-prompt-generator
openOutpaint
openOutpaint-webUI-extension
openpose-editor
sd-dynamic-prompts
sd-dynamic-thresholding
sd-webui-agentattention
sd-webui-cads
sd-webui-cd-tuner
sd-webui-cleaner
sd-webui-clear-screen
sd-webui-color-correction-extras
sd-webui-color-enhance sd-webui-controlnet
sd-webui-creaprompt
sd-webui-decadetw-auto-prompt-llm
sd-webui-deoldify
sd-webui-detail-daemon sd-webui-diffusion-cg
sd-webui-facefusion
sd-webui-faceswaplab
sd-webui-freeu
sd-webui-incantations
sd-webui-inpaint-anything
sd-webui-inpaint-background
sd-webui-kohya-hiresfix
sd-webui-lama-cleaner-masked-content
sd-webui-lcm
sd-webui-llul
sd-webui-negpip
sd-webui-neutral-prompt
sd-webui-old-photo-restoration
sd-webui-prompt-all-in-one
sd-webui-ranbooru
sd-webui-reactor
sd-webui-regional-prompter
sd-webui-roop
sd-webui-rpg-diffusionmaster
sd-webui-segment-anything
sd-webui-semantic-guidance
sd-webui-temporal
stable-diffusion-NPW
stable-diffusion-webui-Prompt_Generator
stable-diffusion-webui-conditioning-highres-fix
stable-diffusion-webui-promptgen
stable-diffusion-webui-randomize
stable-diffusion-webui-rembg
stable-diffusion-webui-sonar
stable-diffusion-webui-tokenizer
stable-diffusion-webui-wildcards
ultimate-upscale-for-automatic1111