search

s9roll7 / animatediff-cli-prompt-travel

animatediff prompt travel
Apache License 2.0
1.19k stars 105 forks source link

Multi Controlnet killed #183

Closed yukselyusuf closed 10 months ago

yukselyusuf commented 10 months ago

im using multi controlnet (depth, canny, openpose,)and I have issue.

versions: ubuntu:20.04.6 python: 3.10.13 diffusers : 0.23.1

Screenshot 2023-12-01 at 09 59 07

my config :


{
    "name": "sample",
    "path": "models/sd/xxmix_realistic.safetensors",
    "vae_path": "",
    "motion_module": "models/motion-module/mm_sd_v15_v2.ckpt",
    "lcm_map": {
        "enable": false,
        "start_scale": 0.15,
        "end_scale": 0.75,
        "gradient_start": 0.2,
        "gradient_end": 0.75
    },
    "compile": false,
    "tensor_interpolation_slerp": true,
    "seed": [
        -1
    ],
    "scheduler": "k_dpmpp_sde",
    "steps": 20,
    "guidance_scale": 7.0,
    "unet_batch_size": 1,
    "clip_skip": 2,
    "prompt_fixed_ratio": 0.8,
    "head_prompt": "black hair, still background, marble queen in the style of M4rbleSCNEW",
    "prompt_map":{
        "0": "a queen dancing,M4rbleSCNEW,still background"
    },
    "tail_prompt": "masterpiece,best quality,still background",
    "n_prompt": [
        "(worst quality, low quality:1.4),nudity,border,text, patreon,bed,white background,((monochrome)),sketch,,7 arms,8 arms,4 arms"
    ],
    "is_single_prompt_mode": false,
    "lora_map": {"models/lora/marble.safetensors":0.7},
    "motion_lora_map": {},
    "ip_adapter_map": {
        "enable": false,
        "input_image_dir": "../stylize/2023-11-30T11-50-16-sample-mistoonanime_v20/00_ipadapter",
        "prompt_fixed_ratio": 0.5,
        "save_input_image": true,
        "resized_to_square": false,
        "scale": 0.5,
        "is_full_face": false,
        "is_plus_face": false,
        "is_plus": true,
        "is_light": false
    },
    "img2img_map": {
        "enable": false,
        "init_img_dir": "../stylize/2023-11-30T11-50-16-sample-mistoonanime_v20/00_img2img",
        "save_init_image": true,
        "denoising_strength": 0.7
    },
    "region_map": {},
    "controlnet_map": {
        "input_image_dir": "../stylize/2023-11-30T11-50-16-sample-mistoonanime_v20/00_controlnet_image",
        "max_samples_on_vram": 200,
        "max_models_on_vram": 3,
        "save_detectmap": true,
        "preprocess_on_gpu": true,
        "is_loop": false,
        "controlnet_tile": {
            "enable": false,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_ip2p": {
            "enable": false,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 0.5,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_lineart_anime": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_openpose": {
            "enable": true,
            "use_preprocessor": false,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_softedge": {
            "enable": true,
            "use_preprocessor": true,
            "preprocessor": {
                "type": "softedge_pidsafe",
                "param": {}
            },
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_shuffle": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_depth": {
            "enable": true,
            "use_preprocessor": false,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_canny": {
            "enable": false,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_inpaint": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_lineart": {
            "enable": false,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_mlsd": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_normalbae": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_scribble": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_seg": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "qr_code_monster_v1": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "qr_code_monster_v2": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_mediapipe_face": {
            "enable": true,
            "use_preprocessor": true,
            "guess_mode": false,
            "controlnet_conditioning_scale": 1.0,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0,
            "control_scale_list": []
        },
        "controlnet_ref": {
            "enable": false,
            "ref_image": "ref_image/ref_sample.png",
            "attention_auto_machine_weight": 0.3,
            "gn_auto_machine_weight": 0.3,
            "style_fidelity": 0.5,
            "reference_attn": true,
            "reference_adain": false,
            "scale_pattern": [
                1.0
            ]
        }
    },
    "upscale_config": {
        "scheduler": "k_dpmpp_sde",
        "steps": 20,
        "strength": 0.5,
        "guidance_scale": 10,
        "controlnet_tile": {
            "enable": true,
            "controlnet_conditioning_scale": 1.0,
            "guess_mode": false,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0
        },
        "controlnet_line_anime": {
            "enable": false,
            "controlnet_conditioning_scale": 1.0,
            "guess_mode": false,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0
        },
        "controlnet_ip2p": {
            "enable": false,
            "controlnet_conditioning_scale": 0.5,
            "guess_mode": false,
            "control_guidance_start": 0.0,
            "control_guidance_end": 1.0
        },
        "controlnet_ref": {
            "enable": false,
            "use_frame_as_ref_image": false,
            "use_1st_frame_as_ref_image": false,
            "ref_image": "ref_image/path_to_your_ref_img.jpg",
            "attention_auto_machine_weight": 1.0,
            "gn_auto_machine_weight": 1.0,
            "style_fidelity": 0.25,
            "reference_attn": true,
            "reference_adain": false
        }
    },
    "stylize_config": {
        "original_video": {
            "path": "tiktok1.mp4",
            "aspect_ratio": -1.0,
            "offset": 0
        },
        "create_mask": [
            "person"
        ],
        "composite": {
            "fg_list": [
                {
                    "path": " absolute path to frame dir ",
                    "mask_path": " absolute path to mask dir (this is optional) ",
                    "mask_prompt": "person"
                },
                {
                    "path": " absolute path to frame dir ",
                    "mask_path": " absolute path to mask dir (this is optional) ",
                    "mask_prompt": "cat"
                }
            ],
            "bg_frame_dir": "Absolute path to the BG frame directory",
            "hint": ""
        },
        "1": {
            "width": 576,
            "height": 1024,
            "length": 313,
            "context": 16,
            "overlap": 4,
            "stride": 0
        },
        "0": {
            "steps": 20,
            "guidance_scale": 10.0,
            "width": 576,
            "height": 1024,
            "length": 313,
            "context": 8,
            "overlap": 2,
            "stride": 0
            }
    },
    "output": {
        "format": "mp4",
        "fps": 30,
        "encode_param": {
            "crf": 10
        }
    },
    "result": {}
}

animatediff stylize generate stylize/2023-11-30T11-50-16-sample-mistoonanime_v20 -L 16

s9roll7 commented 10 months ago

I saw a report somewhere that the same problem occurs in an environment using conda. please recreate the environment with venv

yukselyusuf commented 10 months ago

I installed it as you said, but it still didn't work. same issue @s9roll7

yukselyusuf commented 10 months ago

It works when you make the first 16 frames, but it doesn't work when you make the whole video.

s9roll7 commented 10 months ago

I'm not sure, but are you saying that things have changed since the first time? Is there room for vram?

yukselyusuf commented 10 months ago

There is no problem when I produce only 16 frames, but when I want to produce the whole video, it gets killed. i have 24 gb ram

whenever i only use single cn it works, but it fails on multi controlnet

yukselyusuf commented 10 months ago
Screenshot 2023-12-04 at 17 10 12

look bro @s9roll7

s9roll7 commented 10 months ago

Probably fixed in the latest version.

zihaog0724 commented 8 months ago

same here.