Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 529, in
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 529, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 343, in train
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 343, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4427, in get_hidden_states
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4427, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'"): 'DistributedDataParallel' object has no attribute 'text_model'
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
steps: 0%| | 0/600 [00:46<?, ?it/s]
[2024-05-12 05:10:55,854] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 30704) of binary: E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\python.exe
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1027, in
main()
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1023, in main
launch_command(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1008, in launch_command
multi_gpu_launcher(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 666, in multi_gpu_launcher
distrib_run.run(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\run.py", line 803, in run
elastic_launch(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 135, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 268, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
Traceback (most recent call last): File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 529, in
Traceback (most recent call last):
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 529, in
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 343, in train
train(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\train_db.py", line 343, in train
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4427, in get_hidden_states
encoder_hidden_states = train_util.get_hidden_states(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\sd-scripts\library\train_util.py", line 4427, in get_hidden_states
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
encoder_hidden_states = text_encoder.text_model.final_layer_norm(encoder_hidden_states)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\nn\modules\module.py", line 1688, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'"): 'DistributedDataParallel' object has no attribute 'text_model'
AttributeError: 'DistributedDataParallel' object has no attribute 'text_model'
steps: 0%| | 0/600 [00:46<?, ?it/s] [2024-05-12 05:10:55,854] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 30704) of binary: E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\python.exe Traceback (most recent call last): File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\runpy.py", line 86, in _run_code exec(code, run_globals) File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1027, in
main()
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1023, in main
launch_command(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 1008, in launch_command
multi_gpu_launcher(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\accelerate\commands\launch.py", line 666, in multi_gpu_launcher
distrib_run.run(args)
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\run.py", line 803, in run
elastic_launch(
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 135, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "E:\AI\AIArchitecture\lora-scripts-v1.7.3\python\lib\site-packages\torch\distributed\launcher\api.py", line 268, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
./sd-scripts/train_db.py FAILED
Failures: [1]: time : 2024-05-12_05:10:55 host : lsy.ft.com rank : 1 (local_rank: 1) exitcode : 1 (pid: 24884) error_file: <N/A> traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
Root Cause (first observed failure): [0]: time : 2024-05-12_05:10:55 host : lsy.ft.com rank : 0 (local_rank: 0) exitcode : 1 (pid: 30704) error_file: <N/A> traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
Looking forward to any help, thanks