Iory1998
commented
1 year ago @bmaltais I have an RTX3090 and I am facing the same exact issue. I did activate venv and run the accelerate config, which saved the settings in the the .cache\huggingface\accelerate\default_config.yaml. The content of the .yaml file is as follows: compute_environment: LOCAL_MACHINE distributed_type: 'NO' downcast_bf16: 'no' gpu_ids: All machine_rank: 0 main_training_function: main mixed_precision: bf16 num_machines: 1 num_processes: 1 rdzv_backend: static same_network: true tpu_env: [] tpu_use_cluster: false tpu_use_sudo: false use_cpu: false
I reinstalled torch wheel manually torch-2.0.1+cu118-cp310-cp310-win_amd64.whl and torchvision-0.15.2+cu118-cp310-cp310-win_amd64.whl without success. I have python 3.10.10 installed, could that be the issue?
Enable xformers for U-Net ╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮ │ D:\Desktop\Dreamthingy\kohya_ss\train_network.py:990 in │
│ │
│ 987 │ args = train_util.read_config_from_file(args, parser) │
│ 988 │ │
│ 989 │ trainer = NetworkTrainer() │
│ ❱ 990 │ trainer.train(args) │
│ 991 │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\train_network.py:222 in train │
│ │
│ 219 │ │ # モデルに xformers とか memory efficient attention を組み込む │
│ 220 │ │ train_util.replace_unet_modules(unet, args.mem_eff_attn, args.xformers, args.sdp │
│ 221 │ │ if torch.version >= "2.0.0": # PyTorch 2.0.0 以上対応のxformersなら以下が使 │
│ ❱ 222 │ │ │ vae.set_use_memory_efficient_attention_xformers(args.xformers) │
│ 223 │ │ │
│ 224 │ │ # 差分追加学習のためにモデルを読み込む │
│ 225 │ │ sys.path.append(os.path.dirname(file)) │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\modeling_utils.py:227 in │
│ set_use_memory_efficient_attention_xformers │
│ │
│ 224 │ │ │
│ 225 │ │ for module in self.children(): │
│ 226 │ │ │ if isinstance(module, torch.nn.Module): │
│ ❱ 227 │ │ │ │ fn_recursive_set_mem_eff(module) │
│ 228 │ │
│ 229 │ def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable │
│ 230 │ │ r""" │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\modeling_utils.py:223 in │
│ fn_recursive_set_mem_eff │
│ │
│ 220 │ │ │ │ module.set_use_memory_efficient_attention_xformers(valid, attention_op) │
│ 221 │ │ │ │
│ 222 │ │ │ for child in module.children(): │
│ ❱ 223 │ │ │ │ fn_recursive_set_mem_eff(child) │
│ 224 │ │ │
│ 225 │ │ for module in self.children(): │
│ 226 │ │ │ if isinstance(module, torch.nn.Module): │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\modeling_utils.py:223 in │
│ fn_recursive_set_mem_eff │
│ │
│ 220 │ │ │ │ module.set_use_memory_efficient_attention_xformers(valid, attention_op) │
│ 221 │ │ │ │
│ 222 │ │ │ for child in module.children(): │
│ ❱ 223 │ │ │ │ fn_recursive_set_mem_eff(child) │
│ 224 │ │ │
│ 225 │ │ for module in self.children(): │
│ 226 │ │ │ if isinstance(module, torch.nn.Module): │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\modeling_utils.py:223 in │
│ fn_recursive_set_mem_eff │
│ │
│ 220 │ │ │ │ module.set_use_memory_efficient_attention_xformers(valid, attention_op) │
│ 221 │ │ │ │
│ 222 │ │ │ for child in module.children(): │
│ ❱ 223 │ │ │ │ fn_recursive_set_mem_eff(child) │
│ 224 │ │ │
│ 225 │ │ for module in self.children(): │
│ 226 │ │ │ if isinstance(module, torch.nn.Module): │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\modeling_utils.py:220 in │
│ fn_recursive_set_mem_eff │
│ │
│ 217 │ │ # gets the message │
│ 218 │ │ def fn_recursive_set_mem_eff(module: torch.nn.Module): │
│ 219 │ │ │ if hasattr(module, "set_use_memory_efficient_attention_xformers"): │
│ ❱ 220 │ │ │ │ module.set_use_memory_efficient_attention_xformers(valid, attention_op) │
│ 221 │ │ │ │
│ 222 │ │ │ for child in module.children(): │
│ 223 │ │ │ │ fn_recursive_set_mem_eff(child) │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\diffusers\models\attention_processor.py:2 │
│ 00 in set_use_memory_efficient_attention_xformers │
│ │
│ 197 │ │ │ │ │ name="xformers", │
│ 198 │ │ │ │ ) │
│ 199 │ │ │ elif not torch.cuda.is_available(): │
│ ❱ 200 │ │ │ │ raise ValueError( │
│ 201 │ │ │ │ │ "torch.cuda.is_available() should be True but is False. xformers' me │
│ 202 │ │ │ │ │ " only available for GPU " │
│ 203 │ │ │ │ ) │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
ValueError: torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only
available for GPU
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ C:\Program │
│ Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64qbz5n2kfra8p0\lib\runpy. │
│ py:196 in _run_module_as_main │
│ │
│ 193 │ main_globals = sys.modules["main"].dict │
│ 194 │ if alter_argv: │
│ 195 │ │ sys.argv[0] = mod_spec.origin │
│ ❱ 196 │ return _run_code(code, main_globals, None, │
│ 197 │ │ │ │ │ "main", mod_spec) │
│ 198 │
│ 199 def run_module(mod_name, init_globals=None, │
│ │
│ C:\Program │
│ Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64qbz5n2kfra8p0\lib\runpy. │
│ py:86 in _run_code │
│ │
│ 83 │ │ │ │ │ loader = loader, │
│ 84 │ │ │ │ │ package = pkg_name, │
│ 85 │ │ │ │ │ spec = mod_spec) │
│ ❱ 86 │ exec(code, run_globals) │
│ 87 │ return run_globals │
│ 88 │
│ 89 def _run_module_code(code, init_globals=None, │
│ │
│ in :7 │
│ │
│ 4 from accelerate.commands.accelerate_cli import main │
│ 5 if name == 'main': │
│ 6 │ sys.argv[0] = re.sub(r'(-script.pyw|.exe)?$', '', sys.argv[0]) │
│ ❱ 7 │ sys.exit(main()) │
│ 8 │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\accelerate\commands\accelerate_cli.py:45 │
│ in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if name == "main": │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py:918 in │
│ launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ D:\Desktop\Dreamthingy\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py:580 in │
│ simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['D:\Desktop\Dreamthingy\kohya_ss\venv\Scripts\python.exe', './train_network.py',
'--v2', '--v_parameterization', '--enable_bucket', '--min_bucket_reso=256', '--max_bucket_reso=2048',
'--pretrained_model_name_or_path=stabilityai/stable-diffusion-2-1',
'--train_data_dir=D:/Desktop/Dreamthingy/LoRASystems/Midjourney/100_mdjAngelLora', '--resolution=512,512',
'--output_dir=D:/Desktop/Dreamthingy/LoRASystems/Finalized', '--network_alpha=128', '--save_model_as=safetensors',
'--network_module=networks.lora', '--text_encoder_lr=5e-05', '--unet_lr=0.0001', '--network_dim=128',
'--output_name=mdjAngelLora_v1.0', '--lr_scheduler_num_cycles=1', '--no_half_vae', '--learning_rate=0.0001',
'--lr_scheduler=constant', '--train_batch_size=3', '--max_train_steps=367', '--save_every_n_epochs=1',
'--mixed_precision=fp16', '--save_precision=fp16', '--seed=1234', '--caption_extension=.txt', '--cache_latents',
'--optimizer_type=AdamW8bit', '--max_data_loader_n_workers=0', '--bucket_reso_steps=64', '--xformers',
'--bucket_no_upscale', '--noise_offset=0.0']' returned non-zero exit status 1.
I've tried as much as I can think of in order to fix ValueError: torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is only available for GPU I redid accelerate config, cleared the cache, and I just don't know what to try anymore. Any help is appreciated.