wh1sperC
commented
1 year ago - PY_ARGS=
- set -o pipefail ++ echo configs/motrv2.args ++ sed -e s/configs/exps/g ++ sed -e 's/.args$//g'
- OUTPUT_BASE=exps/motrv2
- mkdir -p exps/motrv2 ++ seq 100
- for RUN in $(seq 100)
- ls exps/motrv2
- grep run1
- OUTPUT_DIR=exps/motrv2/run1
- mkdir exps/motrv2/run1
- break
- echo 'Backing up to log dir: exps/motrv2/run1' Backing up to log dir: exps/motrv2/run1
- rmpyc ++ find -name pycache
- rm -rf ++ find -name '*.pyc'
- rm -rf
- cp -r models datasets util main.py engine.py submit_dance.py configs/motrv2.args exps/motrv2/run1
- echo ' ...Done' ...Done ++ cat configs/motrv2.args
- args='--meta_arch motr --dataset_file e2e_dance --epoch 5 --with_box_refine --lr_drop 4 --lr 2e-4 --lr_backbone 2e-5 --pretrained mot/r50_deformable_detr_plus_iterative_bbox_refinement-checkpoint.pth --batch_size 128 --sample_mode random_interval --sample_interval 10 --sampler_lengths 5 --merger_dropout 0 --dropout 0 --random_drop 0.1 --fp_ratio 0.3 --query_interaction_layer QIMv2 --query_denoise 0.05 --num_queries 10 --append_crowd --det_db det_db_motrv2.json --use_checkpoint '
- pushd exps/motrv2/run1 ~/MOTRv2/exps/motrv2/run1 ~/MOTRv2
- trap cleanup EXIT
- echo 'Logging git status' Logging git status
- git status
- git rev-parse HEAD
- git diff
- echo
- echo ' ...Done' ...Done
- OMP_NUM_THREADS=1
- python -m torch.distributed.launch --nproc_per_node=4 --use_env main.py --meta_arch motr --dataset_file e2e_dance --epoch 5 --with_box_refine --lr_drop 4 --lr 2e-4 --lr_backbone 2e-5 --pretrained mot/r50_deformable_detr_plus_iterative_bbox_refinement-checkpoint.pth --batch_size 128 --sample_mode random_interval --sample_interval 10 --sampler_lengths 5 --merger_dropout 0 --dropout 0 --random_drop 0.1 --fp_ratio 0.3 --query_interaction_layer QIMv2 --query_denoise 0.05 --num_queries 10 --append_crowd --det_db det_db_motrv2.json --use_checkpoint --output_dir .
- tee -a output.log
| distributed init (rank 2): env://
| distributed init (rank 0): env://
| distributed init (rank 1): env://
| distributed init (rank 3): env://
Traceback (most recent call last):
File "/data1/dcy/anaconda3/envs/motrv2/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/data1/dcy/anaconda3/envs/motrv2/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/data1/dcy/anaconda3/envs/motrv2/lib/python3.7/site-packages/torch/distributed/launch.py", line 260, in
main() File "/data1/dcy/anaconda3/envs/motrv2/lib/python3.7/site-packages/torch/distributed/launch.py", line 256, in main cmd=cmd) subprocess.CalledProcessError: Command '['/data1/dcy/anaconda3/envs/motrv2/bin/python', '-u', 'main.py', '--meta_arch', 'motr', '--dataset_file', 'e2e_dance', '--epoch', '5', '--with_box_refine', '--lr_drop', '4', '--lr', '2e-4', '--lr_backbone', '2e-5', '--pretrained', 'mot/r50_deformable_detr_plus_iterative_bbox_refinement-checkpoint.pth', '--batch_size', '128', '--sample_mode', 'random_interval', '--sample_interval', '10', '--sampler_lengths', '5', '--merger_dropout', '0', '--dropout', '0', '--random_drop', '0.1', '--fp_ratio', '0.3', '--query_interaction_layer', 'QIMv2', '--query_denoise', '0.05', '--num_queries', '10', '--append_crowd', '--det_db', 'det_db_motrv2.json', '--use_checkpoint', '--output_dir', '.']' died with <Signals.SIGSEGV: 11>. - cleanup
- echo 'Packing source code' Packing source code
- rmpyc ++ find -name pycache
- rm -rf ./pycache ./datasets/pycache ./models/pycache ./models/ops/functions/pycache ./models/ops/modules/pycache ./models/structures/pycache ./util/pycache ++ find -name '*.pyc'
- rm -rf
- echo ' ...Done' ...Done