./dist_train.sh configs/coco/mask_rcnn_internimage_b_fpn_3x_coco.py 1 --local-rank 0
/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py:181: FutureWarning: The module torch.distributed.launch is deprecated
and will be removed in future. Use torchrun.
Note that --use-env is set by default in torchrun.
If your script expects --local-rank argument to be set, please
change it to read from os.environ['LOCAL_RANK'] instead. See
https://pytorch.org/docs/stable/distributed.html#launch-utility for
further instructions
warnings.warn(
/meta/cash/llm/InternImage/detection/ops_dcnv3/modules/dcnv3.py:20: UserWarning: Now, we support DCNv4 in InternImage.
warnings.warn('Now, we support DCNv4 in InternImage.')
/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmdet/utils/setup_env.py:38: UserWarning: Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
warnings.warn(
/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
warnings.warn(
Traceback (most recent call last):
File "/meta/cash/llm/InternImage/detection/./train.py", line 247, in
main()
File "/meta/cash/llm/InternImage/detection/./train.py", line 183, in main
cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmcv/utils/config.py", line 596, in dump
f.write(self.pretty_text)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmcv/utils/config.py", line 508, in prettytext
text, = FormatCode(text, style_config=yapf_style, verify=True)
TypeError: FormatCode() got an unexpected keyword argument 'verify'
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 98243) of binary: /home/ahs/anaconda3/envs/llm/bin/python
Traceback (most recent call last):
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 196, in
main()
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 192, in main
launch(args)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 177, in launch
run(args)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/run.py", line 785, in run
elastic_launch(
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 134, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
./dist_train.sh configs/coco/mask_rcnn_internimage_b_fpn_3x_coco.py 1 --local-rank 0 /home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py:181: FutureWarning: The module torch.distributed.launch is deprecated and will be removed in future. Use torchrun. Note that --use-env is set by default in torchrun. If your script expects
--local-rank
argument to be set, please change it to read fromos.environ['LOCAL_RANK']
instead. See https://pytorch.org/docs/stable/distributed.html#launch-utility for further instructionswarnings.warn( /meta/cash/llm/InternImage/detection/ops_dcnv3/modules/dcnv3.py:20: UserWarning: Now, we support DCNv4 in InternImage. warnings.warn('Now, we support DCNv4 in InternImage.') /home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmdet/utils/setup_env.py:38: UserWarning: Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. warnings.warn( /home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmdet/utils/setup_env.py:48: UserWarning: Setting MKL_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. warnings.warn( Traceback (most recent call last): File "/meta/cash/llm/InternImage/detection/./train.py", line 247, in
main()
File "/meta/cash/llm/InternImage/detection/./train.py", line 183, in main
cfg.dump(osp.join(cfg.work_dir, osp.basename(args.config)))
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmcv/utils/config.py", line 596, in dump
f.write(self.pretty_text)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/mmcv/utils/config.py", line 508, in prettytext
text, = FormatCode(text, style_config=yapf_style, verify=True)
TypeError: FormatCode() got an unexpected keyword argument 'verify'
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 98243) of binary: /home/ahs/anaconda3/envs/llm/bin/python
Traceback (most recent call last):
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 196, in
main()
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 192, in main
launch(args)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launch.py", line 177, in launch
run(args)
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/run.py", line 785, in run
elastic_launch(
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 134, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/ahs/anaconda3/envs/llm/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: