Traceback (most recent call last):
File "main_group_vit.py", line 460, in
main()
File "main_group_vit.py", line 455, in main
train(cfg)
File "main_group_vit.py", line 115, in train
model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.train.amp_opt_level)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/frontend.py", line 357, in initialize
return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/_initialize.py", line 241, in _initialize
handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/amp.py", line 101, in init
try_caching, verbose)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/wrap.py", line 33, in cached_cast
if not utils.has_func(mod, fn):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/utils.py", line 132, in has_func
if isinstance(mod, torch.nn.backends.backend.FunctionBackend):
AttributeError: module 'torch.nn' has no attribute 'backends'
Traceback (most recent call last):
File "main_group_vit.py", line 460, in
main()
File "main_group_vit.py", line 455, in main
train(cfg)
File "main_group_vit.py", line 115, in train
model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.train.amp_opt_level)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/frontend.py", line 357, in initialize
return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/_initialize.py", line 241, in _initialize
handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/amp.py", line 101, in init
try_caching, verbose)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/wrap.py", line 33, in cached_cast
if not utils.has_func(mod, fn):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/utils.py", line 132, in has_func
if isinstance(mod, torch.nn.backends.backend.FunctionBackend):
AttributeError: module 'torch.nn' has no attribute 'backends'
Killing subprocess 72008
Killing subprocess 72009
Killing subprocess 72013
Traceback (most recent call last):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 340, in
main()
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 326, in main
sigkill_handler(signal.SIGTERM, None) # not coming back
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 301, in sigkill_handler
raise subprocess.CalledProcessError(returncode=last_return_code, cmd=cmd)
subprocess.CalledProcessError: Command '['/data2/shengwang_anaconda/envs/groupvit/bin/python', '-u', 'main_group_vit.py', '--local_rank=2', '--cfg', 'configs/group_vit_gcc_yfcc_30e.yml']' returned non-zero exit status 1.
Traceback (most recent call last): File "main_group_vit.py", line 460, in
main()
File "main_group_vit.py", line 455, in main
train(cfg)
File "main_group_vit.py", line 115, in train
model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.train.amp_opt_level)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/frontend.py", line 357, in initialize
return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/_initialize.py", line 241, in _initialize
handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/amp.py", line 101, in init
try_caching, verbose)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/wrap.py", line 33, in cached_cast
if not utils.has_func(mod, fn):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/utils.py", line 132, in has_func
if isinstance(mod, torch.nn.backends.backend.FunctionBackend):
AttributeError: module 'torch.nn' has no attribute 'backends'
Traceback (most recent call last):
File "main_group_vit.py", line 460, in
main()
File "main_group_vit.py", line 455, in main
train(cfg)
File "main_group_vit.py", line 115, in train
model, optimizer = amp.initialize(model, optimizer, opt_level=cfg.train.amp_opt_level)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/frontend.py", line 357, in initialize
return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/_initialize.py", line 241, in _initialize
handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2))
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/amp.py", line 101, in init
try_caching, verbose)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/wrap.py", line 33, in cached_cast
if not utils.has_func(mod, fn):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/apex/amp/utils.py", line 132, in has_func
if isinstance(mod, torch.nn.backends.backend.FunctionBackend):
AttributeError: module 'torch.nn' has no attribute 'backends'
Killing subprocess 72008
Killing subprocess 72009
Killing subprocess 72013
Traceback (most recent call last):
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 340, in
main()
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 326, in main
sigkill_handler(signal.SIGTERM, None) # not coming back
File "/data2/shengwang_anaconda/envs/groupvit/lib/python3.7/site-packages/torch/distributed/launch.py", line 301, in sigkill_handler
raise subprocess.CalledProcessError(returncode=last_return_code, cmd=cmd)
subprocess.CalledProcessError: Command '['/data2/shengwang_anaconda/envs/groupvit/bin/python', '-u', 'main_group_vit.py', '--local_rank=2', '--cfg', 'configs/group_vit_gcc_yfcc_30e.yml']' returned non-zero exit status 1.