AUTOMATIC1111 / stable-diffusion-webui

Stable Diffusion web UI
GNU Affero General Public License v3.0
139.23k stars 26.42k forks source link

[Bug]: 1.7.0 crashing when loading model on fresh install #14456

Closed davecazz closed 7 months ago

davecazz commented 8 months ago

Checklist

What happened?

I am using a A10 instance on azure running windows 11 and attempting to install and run automatic1111 but its crashing when trying to load v1-5-pruned-emaonly.safetensors

tried reinstalling many times, deleting the venv folder, cloning the repo to different drives. tried switching the 1.5 model to known good one in case it was corrupt. I switched it out with the aurora checkpoint and it was able to load the model, but then it crashed when I tried to generate a simple image.

Steps to reproduce the problem

just load the web ui and get it to load the v1-5-pruned-emaonly.safetensors checkpoint

What should have happened?

I should have been able to install and run auto1111

What browsers do you use to access the UI ?

Microsoft Edge

Sysinfo

{ "Platform": "Windows-10-10.0.22631-SP0", "Python": "3.10.6", "Version": "v1.7.0", "Commit": "cf2772fab0af5573da775e7437e6acdca424f26e", "Script path": "E:\git\stable-diffusion-webui", "Data path": "E:\git\stable-diffusion-webui", "Extensions dir": "E:\git\stable-diffusion-webui\extensions", "Checksum": "0d3f1d8cc4503859690cf6ae09694d9c61a103c3779c599651fb6084fa3e879c", "Commandline": [ "launch.py", "--xformers", "--listen", "--no-half" ], "Torch env info": { "torch_version": "2.0.1+cu118", "is_debug_build": "False", "cuda_compiled_version": "11.8", "gcc_version": null, "clang_version": null, "cmake_version": null, "os": "Microsoft Windows 11 Pro", "libc_version": "N/A", "python_version": "3.10.6 (tags/v3.10.6:9c7b4bd, Aug 1 2022, 21:53:49) [MSC v.1932 64 bit (AMD64)] (64-bit runtime)", "python_platform": "Windows-10-10.0.22631-SP0", "is_cuda_available": "True", "cuda_runtime_version": null, "cuda_module_loading": "LAZY", "nvidia_driver_version": "537.13", "nvidia_gpu_models": "GPU 0: NVIDIA A10-4Q", "cudnn_version": null, "pip_version": "pip3", "pip_packages": [ "numpy==1.23.5", "open-clip-torch==2.20.0", "pytorch-lightning==1.9.4", "torch==2.0.1+cu118", "torchdiffeq==0.2.3", "torchmetrics==1.2.1", "torchsde==0.2.6", "torchvision==0.15.2+cu118" ], "conda_packages": null, "hip_compiled_version": "N/A", "hip_runtime_version": "N/A", "miopen_runtime_version": "N/A", "caching_allocator_config": "", "is_xnnpack_available": "True", "cpu_info": [ "Architecture=9", "CurrentClockSpeed=3194", "DeviceID=CPU0", "Family=2", "L2CacheSize=", "L2CacheSpeed=", "Manufacturer=AuthenticAMD", "MaxClockSpeed=3194", "Name=AMD EPYC 74F3 24-Core Processor ", "ProcessorType=3", "Revision=257" ] }, "Exceptions": [], "CPU": { "model": "AMD64 Family 25 Model 1 Stepping 1, AuthenticAMD", "count logical": 6, "count physical": 3 }, "RAM": { "total": "55GB", "used": "14GB", "free": "40GB" }, "Extensions": [], "Inactive extensions": [], "Environment": { "COMMANDLINE_ARGS": "--xformers --listen --no-half", "GRADIO_ANALYTICS_ENABLED": "False" }, "Config": { "samples_save": true, "samples_format": "png", "samples_filename_pattern": "", "save_images_add_number": true, "save_images_replace_action": "Replace", "grid_save": true, "grid_format": "png", "grid_extended_filename": false, "grid_only_if_multiple": true, "grid_prevent_empty_spots": false, "grid_zip_filename_pattern": "", "n_rows": -1, "font": "", "grid_text_active_color": "#000000", "grid_text_inactive_color": "#999999", "grid_background_color": "#ffffff", "save_images_before_face_restoration": false, "save_images_before_highres_fix": false, "save_images_before_color_correction": false, "save_mask": false, "save_mask_composite": false, "jpeg_quality": 80, "webp_lossless": false, "export_for_4chan": true, "img_downscale_threshold": 4.0, "target_side_length": 4000, "img_max_size_mp": 200, "use_original_name_batch": true, "use_upscaler_name_as_suffix": false, "save_selected_only": true, "save_init_img": false, "temp_dir": "", "clean_temp_dir_at_start": false, "save_incomplete_images": false, "notification_audio": true, "notification_volume": 100, "outdir_samples": "", "outdir_txt2img_samples": "outputs/txt2img-images", "outdir_img2img_samples": "outputs/img2img-images", "outdir_extras_samples": "outputs/extras-images", "outdir_grids": "", "outdir_txt2img_grids": "outputs/txt2img-grids", "outdir_img2img_grids": "outputs/img2img-grids", "outdir_save": "log/images", "outdir_init_images": "outputs/init-images", "save_to_dirs": true, "grid_save_to_dirs": true, "use_save_to_dirs_for_ui": false, "directories_filename_pattern": "[date]", "directories_max_prompt_words": 8, "ESRGAN_tile": 192, "ESRGAN_tile_overlap": 8, "realesrgan_enabled_models": [ "R-ESRGAN 4x+", "R-ESRGAN 4x+ Anime6B" ], "upscaler_for_img2img": null, "face_restoration": false, "face_restoration_model": "CodeFormer", "code_former_weight": 0.5, "face_restoration_unload": false, "auto_launch_browser": "Local", "enable_console_prompts": false, "show_warnings": false, "show_gradio_deprecation_warnings": true, "memmon_poll_rate": 8, "samples_log_stdout": false, "multiple_tqdm": true, "print_hypernet_extra": false, "list_hidden_files": true, "disable_mmap_load_safetensors": false, "hide_ldm_prints": true, "dump_stacks_on_signal": false, "api_enable_requests": true, "api_forbid_local_requests": true, "api_useragent": "", "unload_models_when_training": false, "pin_memory": false, "save_optimizer_state": false, "save_training_settings_to_txt": true, "dataset_filename_word_regex": "", "dataset_filename_join_string": " ", "training_image_repeats_per_epoch": 1, "training_write_csv_every": 500, "training_xattention_optimizations": false, "training_enable_tensorboard": false, "training_tensorboard_save_images": false, "training_tensorboard_flush_every": 120, "sd_model_checkpoint": "v1-5-pruned-emaonly.safetensors [6ce0161689]", "sd_checkpoints_limit": 1, "sd_checkpoints_keep_in_cpu": true, "sd_checkpoint_cache": 0, "sd_unet": "Automatic", "enable_quantization": false, "enable_emphasis": true, "enable_batch_seeds": true, "comma_padding_backtrack": 20, "CLIP_stop_at_last_layers": 1, "upcast_attn": false, "randn_source": "GPU", "tiling": false, "hires_fix_refiner_pass": "second pass", "sdxl_crop_top": 0, "sdxl_crop_left": 0, "sdxl_refiner_low_aesthetic_score": 2.5, "sdxl_refiner_high_aesthetic_score": 6.0, "sd_vae_checkpoint_cache": 0, "sd_vae": "Automatic", "sd_vae_overrides_per_model_preferences": true, "auto_vae_precision": true, "sd_vae_encode_method": "Full", "sd_vae_decode_method": "Full", "inpainting_mask_weight": 1.0, "initial_noise_multiplier": 1.0, "img2img_extra_noise": 0.0, "img2img_color_correction": false, "img2img_fix_steps": false, "img2img_background_color": "#ffffff", "img2img_editor_height": 720, "img2img_sketch_default_brush_color": "#ffffff", "img2img_inpaint_mask_brush_color": "#ffffff", "img2img_inpaint_sketch_default_brush_color": "#ffffff", "return_mask": false, "return_mask_composite": false, "img2img_batch_show_results_limit": 32, "cross_attention_optimization": "Automatic", "s_min_uncond": 0.0, "token_merging_ratio": 0.0, "token_merging_ratio_img2img": 0.0, "token_merging_ratio_hr": 0.0, "pad_cond_uncond": false, "persistent_cond_cache": true, "batch_cond_uncond": true, "use_old_emphasis_implementation": false, "use_old_karras_scheduler_sigmas": false, "no_dpmpp_sde_batch_determinism": false, "use_old_hires_fix_width_height": false, "dont_fix_second_order_samplers_schedule": false, "hires_fix_use_firstpass_conds": false, "use_old_scheduling": false, "interrogate_keep_models_in_memory": false, "interrogate_return_ranks": false, "interrogate_clip_num_beams": 1, "interrogate_clip_min_length": 24, "interrogate_clip_max_length": 48, "interrogate_clip_dict_limit": 1500, "interrogate_clip_skip_categories": [], "interrogate_deepbooru_score_threshold": 0.5, "deepbooru_sort_alpha": true, "deepbooru_use_spaces": true, "deepbooru_escape": true, "deepbooru_filter_tags": "", "extra_networks_show_hidden_directories": true, "extra_networks_dir_button_function": false, "extra_networks_hidden_models": "When searched", "extra_networks_default_multiplier": 1.0, "extra_networks_card_width": 0, "extra_networks_card_height": 0, "extra_networks_card_text_scale": 1.0, "extra_networks_card_show_desc": true, "extra_networks_card_order_field": "Path", "extra_networks_card_order": "Ascending", "extra_networks_add_text_separator": " ", "ui_extra_networks_tab_reorder": "", "textual_inversion_print_at_load": false, "textual_inversion_add_hashes_to_infotext": true, "sd_hypernetwork": "None", "keyedit_precision_attention": 0.1, "keyedit_precision_extra": 0.05, "keyedit_delimiters": ".,\/!?%^*;:{}=`~() ", "keyedit_delimiters_whitespace": [ "Tab", "Carriage Return", "Line Feed" ], "keyedit_move": true, "disable_token_counters": false, "return_grid": true, "do_not_show_images": false, "js_modal_lightbox": true, "js_modal_lightbox_initially_zoomed": true, "js_modal_lightbox_gamepad": false, "js_modal_lightbox_gamepad_repeat": 250, "gallery_height": "", "compact_prompt_box": false, "samplers_in_dropdown": true, "dimensions_and_batch_together": true, "sd_checkpoint_dropdown_use_short": false, "hires_fix_show_sampler": false, "hires_fix_show_prompts": false, "txt2img_settings_accordion": false, "img2img_settings_accordion": false, "localization": "None", "quicksettings_list": [ "sd_model_checkpoint" ], "ui_tab_order": [], "hidden_tabs": [], "ui_reorder_list": [], "gradio_theme": "Default", "gradio_themes_cache": true, "show_progress_in_title": true, "send_seed": true, "send_size": true, "enable_pnginfo": true, "save_txt": false, "add_model_name_to_info": true, "add_model_hash_to_info": true, "add_vae_name_to_info": true, "add_vae_hash_to_info": true, "add_user_name_to_info": false, "add_version_to_infotext": true, "disable_weights_auto_swap": true, "infotext_skip_pasting": [], "infotext_styles": "Apply if any", "show_progressbar": true, "live_previews_enable": true, "live_previews_image_format": "png", "show_progress_grid": true, "show_progress_every_n_steps": 10, "show_progress_type": "Approx NN", "live_preview_allow_lowvram_full": false, "live_preview_content": "Prompt", "live_preview_refresh_period": 1000, "live_preview_fast_interrupt": false, "js_live_preview_in_modal_lightbox": false, "hide_samplers": [], "eta_ddim": 0.0, "eta_ancestral": 1.0, "ddim_discretize": "uniform", "s_churn": 0.0, "s_tmin": 0.0, "s_tmax": 0.0, "s_noise": 1.0, "k_sched_type": "Automatic", "sigma_min": 0.0, "sigma_max": 0.0, "rho": 0.0, "eta_noise_seed_delta": 0, "always_discard_next_to_last_sigma": false, "sgm_noise_multiplier": false, "uni_pc_variant": "bh1", "uni_pc_skip_type": "time_uniform", "uni_pc_order": 3, "uni_pc_lower_order_final": true, "postprocessing_enable_in_main_ui": [], "postprocessing_operation_order": [], "upscaling_max_images_in_cache": 5, "postprocessing_existing_caption_action": "Ignore", "disabled_extensions": [], "disable_all_extensions": "none", "restore_config_state_file": "", "sd_checkpoint_hash": "", "ldsr_steps": 100, "ldsr_cached": false, "SCUNET_tile": 256, "SCUNET_tile_overlap": 8, "SWIN_tile": 192, "SWIN_tile_overlap": 8, "hypertile_enable_unet": false, "hypertile_enable_unet_secondpass": false, "hypertile_max_depth_unet": 3, "hypertile_max_tile_unet": 256, "hypertile_swap_size_unet": 3, "hypertile_enable_vae": false, "hypertile_max_depth_vae": 3, "hypertile_max_tile_vae": 128, "hypertile_swap_size_vae": 3 }, "Startup": { "total": 13.600497722625732, "records": { "initial startup": 0.05076932907104492, "prepare environment/checks": 0.009042501449584961, "prepare environment/git version info": 0.05093121528625488, "prepare environment/torch GPU test": 1.8266031742095947, "prepare environment/clone repositores": 0.16065597534179688, "prepare environment/run extensions installers": 0.0, "prepare environment": 2.0792548656463623, "launcher": 0.002000570297241211, "import torch": 2.9173645973205566, "import gradio": 0.9727447032928467, "setup paths": 0.9187030792236328, "import ldm": 0.005001068115234375, "import sgm": 0.0, "initialize shared": 0.37013673782348633, "other imports": 0.5551848411560059, "opts onchange": 0.0, "setup SD model": 0.0010001659393310547, "setup codeformer": 0.11007285118103027, "setup gfpgan": 0.01951766014099121, "set samplers": 0.0, "list extensions": 0.0009999275207519531, "restore config state file": 0.0, "list SD models": 0.00099945068359375, "list localizations": 0.0, "load scripts/custom_code.py": 0.0020003318786621094, "load scripts/img2imgalt.py": 0.0, "load scripts/loopback.py": 0.001001119613647461, "load scripts/outpainting_mk_2.py": 0.0, "load scripts/poor_mans_outpainting.py": 0.0010004043579101562, "load scripts/postprocessing_caption.py": 0.0, "load scripts/postprocessing_codeformer.py": 0.0, "load scripts/postprocessing_create_flipped_copies.py": 0.0009999275207519531, "load scripts/postprocessing_focal_crop.py": 0.00099945068359375, "load scripts/postprocessing_gfpgan.py": 0.0, "load scripts/postprocessing_split_oversized.py": 0.0, "load scripts/postprocessing_upscale.py": 0.0009989738464355469, "load scripts/processing_autosized_crop.py": 0.0, "load scripts/prompt_matrix.py": 0.0010027885437011719, "load scripts/prompts_from_file.py": 0.0, "load scripts/sd_upscale.py": 0.0007178783416748047, "load scripts/xyz_grid.py": 0.0010294914245605469, "load scripts/ldsr_model.py": 0.7009749412536621, "load scripts/lora_script.py": 0.0990910530090332, "load scripts/scunet_model.py": 0.019004106521606445, "load scripts/swinir_model.py": 0.016508102416992188, "load scripts/hotkey_config.py": 0.001001119613647461, "load scripts/extra_options_section.py": 0.0, "load scripts/hypertile_script.py": 0.03302502632141113, "load scripts/hypertile_xyz.py": 0.0009996891021728516, "load scripts/refiner.py": 0.0, "load scripts/seed.py": 0.0, "load scripts": 0.8803544044494629, "load upscalers": 0.009003877639770508, "refresh VAE": 0.0010001659393310547, "refresh textual inversion templates": 0.0, "scripts list_optimizers": 0.0010001659393310547, "scripts list_unets": 0.0, "reload hypernetworks": 0.0030019283294677734, "initialize extra networks": 0.013038873672485352, "scripts before_ui_callback": 0.002002239227294922, "create ui": 0.541426420211792, "gradio launch": 4.172943830490112, "add APIs": 0.004997968673706055, "app_started_callback/lora_script.py": 0.0, "app_started_callback": 0.0 } }, "Packages": [ "absl-py==2.0.0", "accelerate==0.21.0", "addict==2.4.0", "aenum==3.1.15", "aiofiles==23.2.1", "aiohttp==3.9.1", "aiosignal==1.3.1", "altair==5.2.0", "antlr4-python3-runtime==4.9.3", "anyio==3.7.1", "async-timeout==4.0.3", "attrs==23.1.0", "basicsr==1.4.2", "beautifulsoup4==4.12.2", "blendmodes==2022", "cachetools==5.3.2", "certifi==2023.11.17", "charset-normalizer==3.3.2", "clean-fid==0.1.35", "click==8.1.7", "clip==1.0", "colorama==0.4.6", "contourpy==1.2.0", "cycler==0.12.1", "deprecation==2.1.0", "einops==0.4.1", "exceptiongroup==1.2.0", "facexlib==0.3.0", "fastapi==0.94.0", "ffmpy==0.3.1", "filelock==3.13.1", "filterpy==1.4.5", "fonttools==4.47.0", "frozenlist==1.4.1", "fsspec==2023.12.2", "ftfy==6.1.3", "future==0.18.3", "gdown==4.7.1", "gfpgan==1.3.8", "gitdb==4.0.11", "gitpython==3.1.32", "google-auth-oauthlib==1.2.0", "google-auth==2.25.2", "gradio-client==0.5.0", "gradio==3.41.2", "grpcio==1.60.0", "h11==0.12.0", "httpcore==0.15.0", "httpx==0.24.1", "huggingface-hub==0.20.1", "idna==3.6", "imageio==2.33.1", "importlib-metadata==7.0.1", "importlib-resources==6.1.1", "inflection==0.5.1", "jinja2==3.1.2", "jsonmerge==1.8.0", "jsonschema-specifications==2023.12.1", "jsonschema==4.20.0", "kiwisolver==1.4.5", "kornia==0.6.7", "lark==1.1.2", "lazy-loader==0.3", "lightning-utilities==0.10.0", "llvmlite==0.41.1", "lmdb==1.4.1", "lpips==0.1.4", "markdown==3.5.1", "markupsafe==2.1.3", "matplotlib==3.8.2", "mpmath==1.3.0", "multidict==6.0.4", "networkx==3.2.1", "numba==0.58.1", "numpy==1.23.5", "oauthlib==3.2.2", "omegaconf==2.2.3", "open-clip-torch==2.20.0", "opencv-python==4.8.1.78", "orjson==3.9.10", "packaging==23.2", "pandas==2.1.4", "piexif==1.1.3", "pillow==9.5.0", "pip==22.2.1", "platformdirs==4.1.0", "protobuf==3.20.0", "psutil==5.9.5", "pyasn1-modules==0.3.0", "pyasn1==0.5.1", "pydantic==1.10.13", "pydub==0.25.1", "pyparsing==3.1.1", "pysocks==1.7.1", "python-dateutil==2.8.2", "python-multipart==0.0.6", "pytorch-lightning==1.9.4", "pytz==2023.3.post1", "pywavelets==1.5.0", "pyyaml==6.0.1", "realesrgan==0.3.0", "referencing==0.32.0", "regex==2023.12.25", "requests-oauthlib==1.3.1", "requests==2.31.0", "resize-right==0.0.2", "rpds-py==0.16.2", "rsa==4.9", "safetensors==0.3.1", "scikit-image==0.21.0", "scipy==1.11.4", "semantic-version==2.10.0", "sentencepiece==0.1.99", "setuptools==63.2.0", "six==1.16.0", "smmap==5.0.1", "sniffio==1.3.0", "soupsieve==2.5", "starlette==0.26.1", "sympy==1.12", "tb-nightly==2.16.0a20231228", "tensorboard-data-server==0.7.2", "tf-keras-nightly==2.16.0.dev2023122810", "tifffile==2023.12.9", "timm==0.9.2", "tokenizers==0.13.3", "tomesd==0.1.3", "tomli==2.0.1", "toolz==0.12.0", "torch==2.0.1+cu118", "torchdiffeq==0.2.3", "torchmetrics==1.2.1", "torchsde==0.2.6", "torchvision==0.15.2+cu118", "tqdm==4.66.1", "trampoline==0.1.2", "transformers==4.30.2", "typing-extensions==4.9.0", "tzdata==2023.3", "urllib3==2.1.0", "uvicorn==0.25.0", "wcwidth==0.2.12", "websockets==11.0.3", "werkzeug==3.0.1", "xformers==0.0.20", "yapf==0.40.2", "yarl==1.9.4", "zipp==3.17.0" ] }

Console logs

(venv) PS E:\git\stable-diffusion-webui> .\webui-user.bat
venv "E:\git\stable-diffusion-webui\venv\Scripts\Python.exe"
Python 3.10.6 (tags/v3.10.6:9c7b4bd, Aug  1 2022, 21:53:49) [MSC v.1932 64 bit (AMD64)]
Version: v1.7.0
Commit hash: cf2772fab0af5573da775e7437e6acdca424f26e
Launching Web UI with arguments: --xformers --listen --no-half
Style database not found: E:\git\stable-diffusion-webui\styles.csv
Loading weights [6ce0161689] from E:\git\stable-diffusion-webui\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors
Running on local URL:  http://0.0.0.0:7860
Creating model from config: E:\git\stable-diffusion-webui\configs\v1-inference.yaml

To create a public link, set `share=True` in `launch()`.
Startup time: 13.5s (prepare environment: 2.0s, import torch: 2.9s, import gradio: 1.0s, setup paths: 0.9s, initialize shared: 0.4s, other imports: 0.6s, setup codeformer: 0.1s, load scripts: 1.0s, create ui: 0.5s, gradio launch: 4.2s).
Applying attention optimization: xformers... done.
loading stable diffusion model: RuntimeError
Traceback (most recent call last):
  File "C:\Users\dave\.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 973, in _bootstrap
    self._bootstrap_inner()
  File "C:\Users\dave\.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "C:\Users\dave\.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "E:\git\stable-diffusion-webui\modules\initialize.py", line 147, in load_model
    shared.sd_model  # noqa: B018
  File "E:\git\stable-diffusion-webui\modules\shared_items.py", line 128, in sd_model
    return modules.sd_models.model_data.get_sd_model()
  File "E:\git\stable-diffusion-webui\modules\sd_models.py", line 531, in get_sd_model
    load_model()
  File "E:\git\stable-diffusion-webui\modules\sd_models.py", line 681, in load_model
    sd_model.cond_stage_model_empty_prompt = get_empty_cond(sd_model)
  File "E:\git\stable-diffusion-webui\modules\sd_models.py", line 569, in get_empty_cond
    return sd_model.cond_stage_model([""])
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\modules\sd_hijack_clip.py", line 234, in forward
    z = self.process_tokens(tokens, multipliers)
  File "E:\git\stable-diffusion-webui\modules\sd_hijack_clip.py", line 273, in process_tokens
    z = self.encode_with_transformers(tokens)
  File "E:\git\stable-diffusion-webui\modules\sd_hijack_clip.py", line 326, in encode_with_transformers
    outputs = self.wrapped.transformer(input_ids=tokens, output_hidden_states=-opts.CLIP_stop_at_last_layers)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 822, in forward
    return self.text_model(
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 740, in forward
    encoder_outputs = self.encoder(
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 654, in forward
    layer_outputs = encoder_layer(
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 393, in forward
    hidden_states = self.mlp(hidden_states)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\transformers\models\clip\modeling_clip.py", line 350, in forward
    hidden_states = self.fc2(hidden_states)
  File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "E:\git\stable-diffusion-webui\extensions-builtin\Lora\networks.py", line 484, in network_Linear_forward
    network_apply_weights(self)
  File "E:\git\stable-diffusion-webui\extensions-builtin\Lora\networks.py", line 379, in network_apply_weights
    bias_backup = self.bias.to(devices.cpu, copy=True)
RuntimeError: CUDA error: the launch timed out and was terminated
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

Stable diffusion model failed to load
Exception in thread Thread-18 (load_model):
Traceback (most recent call last):
  File "C:\Users\dave\.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "C:\Users\dave\.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "E:\git\stable-diffusion-webui\modules\initialize.py", line 153, in load_model
    devices.first_time_calculation()
  File "E:\git\stable-diffusion-webui\modules\devices.py", line 160, in first_time_calculation
    x = torch.zeros((1, 1)).to(device, dtype)
RuntimeError: CUDA error: the launch timed out and was terminated
CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

Additional information

No response

davecazz commented 8 months ago

just tried 1.6 and similar issue. this exact issue happened on 1.7 with default command line args after fresh install

`PS E:\git\fuck-sd> .\venv\Scripts\Activate.ps1 (venv) PS E:\git\fuck-sd> .\webui-user.bat venv "E:\git\fuck-sd\venv\Scripts\Python.exe" Python 3.10.6 (tags/v3.10.6:9c7b4bd, Aug 1 2022, 21:53:49) [MSC v.1932 64 bit (AMD64)] Version: v1.6.1 Commit hash: 4afaaf8a020c1df457bcf7250cb1c7f609699fa7 Installing torch and torchvision Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu118 Collecting torch==2.0.1 Using cached https://download.pytorch.org/whl/cu118/torch-2.0.1%2Bcu118-cp310-cp310-win_amd64.whl (2619.1 MB) Collecting torchvision==0.15.2 Using cached https://download.pytorch.org/whl/cu118/torchvision-0.15.2%2Bcu118-cp310-cp310-win_amd64.whl (4.9 MB) Collecting sympy Using cached https://download.pytorch.org/whl/sympy-1.12-py3-none-any.whl (5.7 MB) Collecting jinja2 Using cached https://download.pytorch.org/whl/Jinja2-3.1.2-py3-none-any.whl (133 kB) Collecting filelock Using cached filelock-3.13.1-py3-none-any.whl (11 kB) Collecting typing-extensions Using cached typing_extensions-4.9.0-py3-none-any.whl (32 kB) Collecting networkx Using cached networkx-3.2.1-py3-none-any.whl (1.6 MB) Collecting requests Using cached requests-2.31.0-py3-none-any.whl (62 kB) Collecting numpy Using cached numpy-1.26.2-cp310-cp310-win_amd64.whl (15.8 MB) Collecting pillow!=8.3.*,>=5.3.0 Using cached Pillow-10.1.0-cp310-cp310-win_amd64.whl (2.6 MB) Collecting MarkupSafe>=2.0 Using cached https://download.pytorch.org/whl/MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl (17 kB) Collecting idna<4,>=2.5 Using cached idna-3.6-py3-none-any.whl (61 kB) Collecting certifi>=2017.4.17 Using cached certifi-2023.11.17-py3-none-any.whl (162 kB) Collecting urllib3<3,>=1.21.1 Using cached urllib3-2.1.0-py3-none-any.whl (104 kB) Collecting charset-normalizer<4,>=2 Using cached charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl (100 kB) Collecting mpmath>=0.19 Using cached https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB) Installing collected packages: mpmath, urllib3, typing-extensions, sympy, pillow, numpy, networkx, MarkupSafe, idna, filelock, charset-normalizer, certifi, requests, jinja2, torch, torchvision Successfully installed MarkupSafe-2.1.3 certifi-2023.11.17 charset-normalizer-3.3.2 filelock-3.13.1 idna-3.6 jinja2-3.1.2 mpmath-1.3.0 networkx-3.2.1 numpy-1.26.2 pillow-10.1.0 requests-2.31.0 sympy-1.12 torch-2.0.1+cu118 torchvision-0.15.2+cu118 typing-extensions-4.9.0 urllib3-2.1.0

[notice] A new release of pip available: 22.2.1 -> 23.3.2 [notice] To update, run: python.exe -m pip install --upgrade pip Installing clip Installing open_clip Cloning Stable Diffusion into E:\git\fuck-sd\repositories\stable-diffusion-stability-ai... Cloning into 'E:\git\fuck-sd\repositories\stable-diffusion-stability-ai'... remote: Enumerating objects: 580, done. remote: Counting objects: 100% (310/310), done. remote: Compressing objects: 100% (94/94), done. remote: Total 580 (delta 248), reused 216 (delta 216), pack-reused 270 Receiving objects: 100% (580/580), 73.43 MiB | 33.08 MiB/s, done. Resolving deltas: 100% (280/280), done. Cloning Stable Diffusion XL into E:\git\fuck-sd\repositories\generative-models... Cloning into 'E:\git\fuck-sd\repositories\generative-models'... remote: Enumerating objects: 860, done. remote: Counting objects: 100% (513/513), done. remote: Compressing objects: 100% (244/244), done. remote: Total 860 (delta 365), reused 308 (delta 265), pack-reused 347Receiving objects: 99% (852/860), 33.84 MiB | 33.28 MiB/s Receiving objects: 100% (860/860), 42.67 MiB | 33.56 MiB/s, done. Resolving deltas: 100% (436/436), done. Cloning K-diffusion into E:\git\fuck-sd\repositories\k-diffusion... Cloning into 'E:\git\fuck-sd\repositories\k-diffusion'... remote: Enumerating objects: 1329, done. remote: Counting objects: 100% (727/727), done. remote: Compressing objects: 100% (86/86), done. Receiving objects: 100% (1329/1329), 232.86 KiB | 3.95 MiB/s, done. 602

Resolving deltas: 100% (933/933), done. Cloning CodeFormer into E:\git\fuck-sd\repositories\CodeFormer... Cloning into 'E:\git\fuck-sd\repositories\CodeFormer'... remote: Enumerating objects: 594, done. remote: Counting objects: 100% (245/245), done. remote: Compressing objects: 100% (88/88), done. remote: Total 594 (delta 175), reused 173 (delta 157), pack-reused 349 Receiving objects: 100% (594/594), 17.31 MiB | 31.25 MiB/s, done. Resolving deltas: 100% (286/286), done. Cloning BLIP into E:\git\fuck-sd\repositories\BLIP... Cloning into 'E:\git\fuck-sd\repositories\BLIP'... remote: Enumerating objects: 277, done. remote: Counting objects: 100% (165/165), done. remote: Compressing objects: 100% (30/30), done.

Receiving objects: 100% (277/277), 7.03 MiB | 26.48 MiB/s, done. Resolving deltas: 100% (152/152), done. Installing requirements for CodeFormer Installing requirements Launching Web UI with arguments: no module 'xformers'. Processing without... no module 'xformers'. Processing without... No module 'xformers'. Proceeding without it. Downloading: "https://huggingface.co/runwayml/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors" to E:\git\fuck-sd\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors

100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3.97G/3.97G [00:26<00:00, 159MB/s] Calculating sha256 for E:\git\fuck-sd\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors: Running on local URL: http://127.0.0.1:7860

To create a public link, set share=True in launch(). Startup time: 425.6s (prepare environment: 365.5s, launcher: 0.1s, import torch: 5.6s, import gradio: 3.0s, setup paths: 4.3s, initialize shared: 1.1s, other imports: 3.5s, setup codeformer: 0.3s, list SD models: 27.9s, load scripts: 13.3s, create ui: 0.6s, gradio launch: 0.2s). 6ce0161689b3853acaa03779ec93eafe75a02f4ced659bee03f50797806fa2fa Loading weights [6ce0161689] from E:\git\fuck-sd\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors Creating model from config: E:\git\fuck-sd\configs\v1-inference.yaml loading stable diffusion model: RuntimeError Traceback (most recent call last): File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 973, in _bootstrap self._bootstrap_inner() File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner self.run() File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 953, in run self._target(*self._args, **self._kwargs) File "E:\git\fuck-sd\modules\initialize.py", line 147, in load_model shared.sd_model # noqa: B018 File "E:\git\fuck-sd\modules\shared_items.py", line 110, in sd_model return modules.sd_models.model_data.get_sd_model() File "E:\git\fuck-sd\modules\sd_models.py", line 499, in get_sd_model load_model() File "E:\git\fuck-sd\modules\sd_models.py", line 626, in load_model load_model_weights(sd_model, checkpoint_info, state_dict, timer) File "E:\git\fuck-sd\modules\sd_models.py", line 404, in load_model_weights model.logvar = model.logvar.to(devices.device) # fix for training RuntimeError: CUDA error: the launch timed out and was terminated CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.

Stable diffusion model failed to load Applying attention optimization: Doggettx... done. Loading weights [6ce0161689] from E:\git\fuck-sd\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors Exception in thread Thread-27 (load_model): Traceback (most recent call last): File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner self.run() File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 953, in run self._target(*self._args, **self._kwargs) File "E:\git\fuck-sd\modules\initialize.py", line 153, in load_model devices.first_time_calculation() File "E:\git\fuck-sd\modules\devices.py", line 146, in first_time_calculation x = torch.zeros((1, 1)).to(device, dtype) RuntimeError: CUDA error: the launch timed out and was terminated CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.

Creating model from config: E:\git\fuck-sd\configs\v1-inference.yaml loading stable diffusion model: RuntimeError Traceback (most recent call last): File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 973, in _bootstrap self._bootstrap_inner() File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner self.run() File "E:\git\fuck-sd\venv\lib\site-packages\anyio_backends_asyncio.py", line 807, in run result = context.run(func, args) File "E:\git\fuck-sd\venv\lib\site-packages\gradio\utils.py", line 707, in wrapper response = f(args, kwargs) File "E:\git\fuck-sd\modules\ui_extra_networks.py", line 392, in pages_html return refresh() File "E:\git\fuck-sd\modules\ui_extra_networks.py", line 398, in refresh pg.refresh() File "E:\git\fuck-sd\modules\ui_extra_networks_textual_inversion.py", line 13, in refresh sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=True) File "E:\git\fuck-sd\modules\textual_inversion\textual_inversion.py", line 255, in load_textual_inversion_embeddings self.expected_shape = self.get_expected_shape() File "E:\git\fuck-sd\modules\textual_inversion\textual_inversion.py", line 154, in get_expected_shape vec = shared.sd_model.cond_stage_model.encode_embedding_init_text(",", 1) File "E:\git\fuck-sd\modules\shared_items.py", line 110, in sd_model return modules.sd_models.model_data.get_sd_model() File "E:\git\fuck-sd\modules\sd_models.py", line 499, in get_sd_model load_model() File "E:\git\fuck-sd\modules\sd_models.py", line 626, in load_model load_model_weights(sd_model, checkpoint_info, state_dict, timer) File "E:\git\fuck-sd\modules\sd_models.py", line 353, in load_model_weights model.load_state_dict(state_dict, strict=False) File "E:\git\fuck-sd\modules\sd_disable_initialization.py", line 223, in module_load_state_dict = self.replace(torch.nn.Module, 'load_state_dict', lambda *args, *kwargs: load_state_dict(module_load_state_dict, args, kwargs)) File "E:\git\fuck-sd\modules\sd_disable_initialization.py", line 221, in load_state_dict original(module, state_dict, strict=strict) File "E:\git\fuck-sd\venv\lib\site-packages\torch\nn\modules\module.py", line 2041, in load_state_dict raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( RuntimeError: Error(s) in loading state_dict for LatentDiffusion: While copying the parameter named "model.diffusion_model.time_embed.0.weight", whose dimensions in the model are torch.Size([1280, 320]) and whose dimensions in the checkpoint are torch.Size([1280, 320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.time_embed.0.bias", whose dimensions in the model are torch.Size([1280]) and whose dimensions in the checkpoint are torch.Size([1280]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.time_embed.2.weight", whose dimensions in the model are torch.Size([1280, 1280]) and whose dimensions in the checkpoint are torch.Size([1280, 1280]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.time_embed.2.bias", whose dimensions in the model are torch.Size([1280]) and whose dimensions in the checkpoint are torch.Size([1280]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.0.0.weight", whose dimensions in the model are torch.Size([320, 4, 3, 3]) and whose dimensions in the checkpoint are torch.Size([320, 4, 3, 3]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.0.0.bias", whose dimensions in the model are torch.Size([320]) and whose dimensions in the checkpoint are torch.Size([320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.in_layers.2.weight", whose dimensions in the model are torch.Size([320, 320, 3, 3]) and whose dimensions in the checkpoint are torch.Size([320, 320, 3, 3]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.in_layers.2.bias", whose dimensions in the model are torch.Size([320]) and whose dimensions in the checkpoint are torch.Size([320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.emb_layers.1.weight", whose dimensions in the model are torch.Size([320, 1280]) and whose dimensions in the checkpoint are torch.Size([320, 1280]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.emb_layers.1.bias", whose dimensions in the model are torch.Size([320]) and whose dimensions in the checkpoint are torch.Size([320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.out_layers.3.weight", whose dimensions in the model are torch.Size([320, 320, 3, 3]) and whose dimensions in the checkpoint are torch.Size([320, 320, 3, 3]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.0.out_layers.3.bias", whose dimensions in the model are torch.Size([320]) and whose dimensions in the checkpoint are torch.Size([320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.1.proj_in.weight", whose dimensions in the model are torch.Size([320, 320, 1, 1]) and whose dimensions in the checkpoint are torch.Size([320, 320, 1, 1]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.1.proj_in.bias", whose dimensions in the model are torch.Size([320]) and whose dimensions in the checkpoint are torch.Size([320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',). While copying the parameter named "model.diffusion_model.input_blocks.1.1.transformer_blocks.0.attn1.to_q.weight", whose dimensions in the model are torch.Size([320, 320]) and whose dimensions in the checkpoint are torch.Size([320, 320]), an exception occurred : ('CUDA error: the launch timed out and was terminated\nCUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.\nFor debugging consider passing CUDA_LAUNCH_BLOCKING=1.\nCompile with TORCH_USE_CUDA_DSA to enable device-side assertions.\n',).`

davecazz commented 8 months ago

just confirmed that the same thing happened on a brand new instance with a T4 on google cloud. brand new machine, installed latest cuda and nvidia drivers. python 3.10.6 I used powershell to start auto1111. but the 1.5 model failed to load. ius there some dependency that I'm forgetting? really confused because I was able to install it on my alienware laptop with windows 11, and a 2080, it only has 8gb of vram and was struggling on some of the larger models so I want to start using a VM

davecazz commented 8 months ago

I notice in my sysinfo that "cuda_runtime_version": null,"cudnn_version": null. although I ran the follwoing script in python which gave me expected results that it found the A10 on azure.

import torch torch.cuda.is_available() torch.cuda.device_count() torch.cuda.current_device() torch.cuda.device(0) torch.cuda.get_device_name(0)

dairydaddy commented 8 months ago

try deleting the venv directory, then add edit the requirements.txt and change torch to torch==2.0.1

also make sure to check that you don't have torch 2.1 installed. You can use conda list, etc. Deleting the venv folder + uninstalling torch (usually 2.1.2) and installing 2.0.1 fixes it for me. The folder is recreated when webui-user runs.

davecazz commented 8 months ago

Thanks, I'll try that

davecazz commented 8 months ago

getting a little close. although I am just using a venv but not conda, so I deleted the venv which I assume would have uninstalled pytorch, but I did not have it installed in my base python env. is there something at the os level that I would have to remove it?

I no longer get an exception when loading the model but now just get an exception when generating an image

`Loading weights [6ce0161689] from E:\git\stable-diffusion-webui\models\Stable-diffusion\v1-5-pruned-emaonly.safetensors Running on local URL: http://0.0.0.0:7860 Creating model from config: E:\git\stable-diffusion-webui\configs\v1-inference.yaml

To create a public link, set share=True in launch(). Startup time: 337.5s (prepare environment: 319.2s, import torch: 4.4s, import gradio: 1.5s, setup paths: 2.0s, initialize shared: 0.6s, other imports: 1.8s, setup codeformer: 0.4s, load scripts: 1.4s, initialize extra networks: 0.1s, create ui: 1.2s, gradio launch: 4.8s). Applying attention optimization: xformers... done. Model loaded in 93.2s (load weights from disk: 2.3s, create model: 0.3s, apply weights to model: 83.8s, move model to device: 0.4s, load textual inversion embeddings: 2.7s, calculate empty prompt: 3.5s). Exception in thread MemMon: Traceback (most recent call last): File "C:\Users\dave.pyenv\pyenv-win\versions\3.10.6\lib\threading.py", line 1016, in _bootstrap_inner self.run() File "E:\git\stable-diffusion-webui\modules\memmon.py", line 53, in run free, total = self.cuda_mem_get_info() File "E:\git\stable-diffusion-webui\modules\memmon.py", line 34, in cuda_mem_get_info return torch.cuda.mem_get_info(index) File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\cuda\memory.py", line 618, in mem_get_info return torch.cuda.cudart().cudaMemGetInfo(device) RuntimeError: CUDA error: the launch timed out and was terminated CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.

Error completing request Arguments: ('task(0rhfbpn63gb3z8u)', 'something cool', '', [], 20, 'DPM++ 2M Karras', 1, 1, 7, 512, 512, False, 0.7, 2, 'Latent', 0, 0, 0, 'Use same checkpoint', 'Use same sampler', '', '', [], <gradio.routes.Request object at 0x00000256C1D4EB30>, 0, False, '', 0.8, -1, False, -1, 0, 0, 0, False, False, 'positive', 'comma', 0, False, False, 'start', '', 1, '', [], 0, '', [], 0, '', [], True, False, False, False, 0, False) {} Traceback (most recent call last): File "E:\git\stable-diffusion-webui\modules\call_queue.py", line 57, in f res = list(func(*args, kwargs)) File "E:\git\stable-diffusion-webui\modules\call_queue.py", line 36, in f res = func(*args, *kwargs) File "E:\git\stable-diffusion-webui\modules\txt2img.py", line 55, in txt2img processed = processing.process_images(p) File "E:\git\stable-diffusion-webui\modules\processing.py", line 734, in process_images res = process_images_inner(p) File "E:\git\stable-diffusion-webui\modules\processing.py", line 857, in process_images_inner p.setup_conds() File "E:\git\stable-diffusion-webui\modules\processing.py", line 1308, in setup_conds super().setup_conds() File "E:\git\stable-diffusion-webui\modules\processing.py", line 469, in setup_conds self.uc = self.get_conds_with_caching(prompt_parser.get_learned_conditioning, negative_prompts, total_steps, [self.cached_uc], self.extra_network_data) File "E:\git\stable-diffusion-webui\modules\processing.py", line 455, in get_conds_with_caching cache[1] = function(shared.sd_model, required_prompts, steps, hires_steps, shared.opts.use_old_scheduling) File "E:\git\stable-diffusion-webui\modules\prompt_parser.py", line 188, in get_learned_conditioning conds = model.get_learned_conditioning(texts) File "E:\git\stable-diffusion-webui\repositories\stable-diffusion-stability-ai\ldm\models\diffusion\ddpm.py", line 669, in get_learned_conditioning c = self.cond_stage_model(c) File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\nn\modules\module.py", line 1501, in _call_impl return forward_call(args, kwargs) File "E:\git\stable-diffusion-webui\modules\sd_hijack_clip.py", line 234, in forward z = self.process_tokens(tokens, multipliers) File "E:\git\stable-diffusion-webui\modules\sd_hijack_clip.py", line 278, in process_tokens batch_multipliers = torch.asarray(batch_multipliers).to(devices.device) RuntimeError: CUDA error: the launch timed out and was terminated CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.


Traceback (most recent call last): File "E:\git\stable-diffusion-webui\venv\lib\site-packages\gradio\routes.py", line 488, in run_predict output = await app.get_blocks().process_api( File "E:\git\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1431, in process_api result = await self.call_function( File "E:\git\stable-diffusion-webui\venv\lib\site-packages\gradio\blocks.py", line 1103, in call_function prediction = await anyio.to_thread.run_sync( File "E:\git\stable-diffusion-webui\venv\lib\site-packages\anyio\to_thread.py", line 33, in run_sync return await get_asynclib().run_sync_in_worker_thread( File "E:\git\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 877, in run_sync_in_worker_thread return await future File "E:\git\stable-diffusion-webui\venv\lib\site-packages\anyio_backends_asyncio.py", line 807, in run result = context.run(func, args) File "E:\git\stable-diffusion-webui\venv\lib\site-packages\gradio\utils.py", line 707, in wrapper response = f(args, **kwargs) File "E:\git\stable-diffusion-webui\modules\call_queue.py", line 77, in f devices.torch_gc() File "E:\git\stable-diffusion-webui\modules\devices.py", line 61, in torch_gc torch.cuda.empty_cache() File "E:\git\stable-diffusion-webui\venv\lib\site-packages\torch\cuda\memory.py", line 133, in empty_cache torch._C._cuda_emptyCache() RuntimeError: CUDA error: the launch timed out and was terminated CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. Compile with TORCH_USE_CUDA_DSA to enable device-side assertions.`

davecazz commented 8 months ago

I think I know one of the reasons why this isnt going smoothly. I rented a NV6ads_A10_v5 and I didnt realize this is a 1/6th of the gpu which means I only have 4GB VRAM.

davecazz commented 8 months ago

If anyone finds this I fixed it. it was a combination of a few different things, including what I think is a bug in 1.7.0

first thing was setting torch==2.0.1 in the requirements before initial run

then I found that the azure instance I was using was a 6th of a gpu which had only 4GB of VRAM

then I ran into this issue "Cannot copy out of meta tensor; no data!" https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13087

toward the bottom of this issue someone mentioned that they ran into this when upgrading to 1.7.0 so I downgraded to 1.6.1

walla presto, fixed.

soulteary commented 7 months ago

If you want to use the software version supported by the warehouse to complete security, you will encounter many problems without adjusting and updating the project code. @davecazz @dairydaddy

But it's worth using a newer version of the package because it will bring faster speeds

Later, I will complete the complete manual installation tutorial in this ISSUE. You can follow it or make comments and suggestions. https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/14651