When samples_per_gpu=16, lr=0.01, report an error:
Traceback (most recent call last):
File "tools/train.py", line 177, in <module>
main()
File "tools/train.py", line 173, in main
meta=meta)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/apis/train.py", line 150, in train_detector
runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 125, in run
epoch_runner(data_loaders[i], **kwargs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 50, in train
self.run_iter(data_batch, train_mode=True)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 30, in run_iter
**kwargs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/parallel/data_parallel.py", line 67, in train_step
return self.module.train_step(*inputs[0], **kwargs[0])
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/base.py", line 238, in train_step
losses = self(**data)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/fp16/decorators.py", line 51, in new_func
return old_func(*args, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/base.py", line 172, in forward
return self.forward_train(img, img_metas, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/two_stage.py", line 156, in forward_train
proposal_cfg=proposal_cfg)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/base_dense_head.py", line 54, in forward_train
losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/rpn_head.py", line 75, in loss
gt_bboxes_ignore=gt_bboxes_ignore)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/fp16/decorators.py", line 131, in new_func
return old_func(*args, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/anchor_head.py", line 526, in loss
num_total_samples=num_total_samples)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/utils/misc.py", line 54, in multi_apply
return tuple(map(list, zip(*map_results)))
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/anchor_head.py", line 444, in loss_single
cls_score, labels, label_weights, avg_factor=num_total_samples)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/losses/cross_entropy_loss.py", line 215, in forward
**kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/losses/cross_entropy_loss.py", line 86, in binary_cross_entropy
pred, label.float(), pos_weight=class_weight, reduction='none')
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/functional.py", line 2980, in binary_cross_entropy_with_logits
raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))
ValueError: Target size (torch.Size([237120, 1])) must be the same as input size (torch.Size([758784, 1]))
When samples_per_gpu=8, lr=0.005, report an error:
Traceback (most recent call last):
File "tools/train.py", line 177, in <module>
main()
File "tools/train.py", line 173, in main
meta=meta)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/apis/train.py", line 150, in train_detector
runner.run(data_loaders, cfg.workflow, cfg.total_epochs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 125, in run
epoch_runner(data_loaders[i], **kwargs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 50, in train
self.run_iter(data_batch, train_mode=True)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/runner/epoch_based_runner.py", line 30, in run_iter
**kwargs)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/mmcv/parallel/data_parallel.py", line 67, in train_step
return self.module.train_step(*inputs[0], **kwargs[0])
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/base.py", line 238, in train_step
losses = self(**data)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/fp16/decorators.py", line 51, in new_func
return old_func(*args, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/base.py", line 172, in forward
return self.forward_train(img, img_metas, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/detectors/two_stage.py", line 156, in forward_train
proposal_cfg=proposal_cfg)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/base_dense_head.py", line 54, in forward_train
losses = self.loss(*loss_inputs, gt_bboxes_ignore=gt_bboxes_ignore)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/rpn_head.py", line 75, in loss
gt_bboxes_ignore=gt_bboxes_ignore)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/fp16/decorators.py", line 131, in new_func
return old_func(*args, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/anchor_head.py", line 526, in loss
num_total_samples=num_total_samples)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/core/utils/misc.py", line 54, in multi_apply
return tuple(map(list, zip(*map_results)))
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/dense_heads/anchor_head.py", line 444, in loss_single
cls_score, labels, label_weights, avg_factor=num_total_samples)
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/losses/cross_entropy_loss.py", line 215, in forward
**kwargs)
File "/home/underwater/zjy/kesci-2021-underwater-optics/mmdet/models/losses/cross_entropy_loss.py", line 86, in binary_cross_entropy
pred, label.float(), pos_weight=class_weight, reduction='none')
File "/home/underwater/anaconda3/envs/test/lib/python3.7/site-packages/torch/nn/functional.py", line 2980, in binary_cross_entropy_with_logits
raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))
ValueError: Target size (torch.Size([237120, 1])) must be the same as input size (torch.Size([379392, 1]))
Issue Description
I encountered errors while training using
faster_rcnn_r50_fpn_1x_coco.py
.Environment
Reproduce
faster_rcnn_r50_fpn_1x_coco.py
tools/train.py
image_scale=[666, 400]
samples_per_gpu
: 16, 8, 4, 2lr
: 0.01, 0.005, 0.0025Results Run command
Training is successful with batch_size=2 and 4.
When
samples_per_gpu=16, lr=0.01
, report an error:When
samples_per_gpu=8, lr=0.005
, report an error: