I get a Runtime Error when trying to train faster_rcnn detector

I run the following command python tools/train.py configs/faster_rcnn_r50_fpn_1x.py training on retinanet and rpn works fine. The problem remains after rebuilding mmdet Below is the output of the script.

2020-04-07 22:52:23,146 - mmdet - INFO - Environment info:

sys.platform: linux Python: 3.8.2 (default, Feb 26 2020, 22:21:03) [GCC 9.2.1 20200130] CUDA available: True CUDA_HOME: /opt/cuda NVCC: Cuda compilation tools, release 10.2, V10.2.89 GPU 0: GeForce RTX 2070 GPU 1: GeForce GTX 1070 Ti GCC: gcc (Arch Linux 9.3.0-1) 9.3.0 PyTorch: 1.4.1 PyTorch compiling details: PyTorch built with:

GCC 9.3
Intel(R) Math Kernel Library Version 2020.0.1 Product Build 20200208 for Intel(R) 64 architecture applications
Intel(R) MKL-DNN v0.21.1 (Git Hash 7d2fd500bc78936d1d648ca713b901012f470dbc)
OpenMP 201511 (a.k.a. OpenMP 4.5)
NNPACK is enabled
CUDA Runtime 10.2
NVCC architecture flags: -gencode;arch=compute_32,code=sm_32;-gencode;arch=compute_35,code=sm_35;-gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_52,code=sm_52;-gencode;arch=compute_53,code=sm_53;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_62,code=sm_62;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_72,code=sm_72;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_60,code=compute_60;-gencode;arch=compute_61,code=compute_61;-gencode;arch=compute_62,code=compute_62;-gencode;arch=compute_70,code=compute_70;-gencode;arch=compute_72,code=compute_72;-gencode;arch=compute_75,code=compute_75
CuDNN 7.6.5
Magma 2.5.2
Build settings: BLAS=MKL, BUILD_NAMEDTENSOR=OFF, BUILD_TYPE=Release, CXX_FLAGS=-march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fvisibility-inlines-hidden -fopenmp -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -O2 -fPIC -Wno-narrowing -Wall -Wextra -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Wno-stringop-overflow, DISABLE_NUMA=1, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, USE_CUDA=1, USE_EXCEPTION_PTR=1, USE_GFLAGS=ON, USE_GLOG=ON, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, USE_STATIC_DISPATCH=OFF,

TorchVision: 0.5.0 OpenCV: 4.3.0 MMCV: 0.4.2 MMDetection: 1.1.0+365c930 MMDetection Compiler: GCC 9.3 MMDetection CUDA Compiler: 10.2

2020-04-07 22:52:23,147 - mmdet - INFO - Distributed training: False 2020-04-07 22:52:23,147 - mmdet - INFO - Config: /home/kamran/workspace/mmdetection/configs/faster_rcnn_r50_fpn_1x.py

model settings

model = dict( type='FasterRCNN', pretrained='torchvision://resnet50', backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), style='pytorch'), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_scales=[8], anchor_ratios=[0.5, 1.0, 2.0], anchor_strides=[4, 8, 16, 32, 64], target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0], loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', out_size=7, sample_num=2), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='SharedFCBBoxHead', num_fcs=2, in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=81, target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2], reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)))

model training and testing settings

train_cfg = dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_across_levels=False, nms_pre=2000, nms_post=2000, max_num=2000, nms_thr=0.7, min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)) test_cfg = dict( rpn=dict( nms_across_levels=False, nms_pre=1000, nms_post=1000, max_num=1000, nms_thr=0.7, min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_thr=0.5), max_per_img=100)

soft-nms is also supported for rcnn testing

# e.g., nms=dict(type='soft_nms', iou_thr=0.5, min_score=0.05)

)

dataset settings

dataset_type = 'CocoDataset' data_root = 'data/coco/' img_norm_cfg = dict( mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', flip_ratio=0.5), dict(type='Normalize', img_norm_cfg), dict(type='Pad', size_divisor=32), dict(type='DefaultFormatBundle'), dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), ] test_pipeline = [ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(1333, 800), flip=False, transforms=[ dict(type='Resize', keep_ratio=True), dict(type='RandomFlip'), dict(type='Normalize', img_norm_cfg), dict(type='Pad', size_divisor=32), dict(type='ImageToTensor', keys=['img']), dict(type='Collect', keys=['img']), ]) ] data = dict( imgs_per_gpu=2, workers_per_gpu=2, train=dict( type=dataset_type, ann_file=data_root + 'annotations/instances_train2017.json', img_prefix=data_root + 'train2017/', pipeline=train_pipeline), val=dict( type=dataset_type, ann_file=data_root + 'annotations/instances_val2017.json', img_prefix=data_root + 'val2017/', pipeline=test_pipeline), test=dict( type=dataset_type, ann_file=data_root + 'annotations/instances_val2017.json', img_prefix=data_root + 'val2017/', pipeline=test_pipeline)) evaluation = dict(interval=1, metric='bbox')

optimizer

optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))

learning policy

lr_config = dict( policy='step', warmup='linear', warmup_iters=500, warmup_ratio=1.0 / 3, step=[8, 11]) checkpoint_config = dict(interval=1)

yapf:disable

log_config = dict( interval=50, hooks=[ dict(type='TextLoggerHook'),

dict(type='TensorboardLoggerHook')

])

yapf:enable

runtime settings

total_epochs = 12 dist_params = dict(backend='nccl') log_level = 'INFO' work_dir = './work_dirs/faster_rcnn_r50_fpn_1x' load_from = None resume_from = None workflow = [('train', 1)]

2020-04-07 22:52:23,403 - mmdet - INFO - load model from: torchvision://resnet50 2020-04-07 22:52:23,522 - mmdet - WARNING - The model and loaded state dict do not match exactly

unexpected key in source state_dict: fc.weight, fc.bias

2020-04-07 22:52:36,631 - mmdet - INFO - Start running, host: kamran@archbox, work_dir: /home/kamran/workspace/mmdetection/work_dirs/faster_rcnn_r50_fpn_1x 2020-04-07 22:52:36,632 - mmdet - INFO - workflow: [('train', 1)], max: 12 epochs loading annotations into memory... Done (t=10.88s) creating index... index created! Traceback (most recent call last): File "tools/train.py", line 142, in main() File "tools/train.py", line 131, in main train_detector( File "/home/kamran/workspace/mmdetection/mmdet/apis/train.py", line 104, in train_detector _non_dist_train( File "/home/kamran/workspace/mmdetection/mmdet/apis/train.py", line 242, in _non_dist_train runner.run(data_loaders, cfg.workflow, cfg.total_epochs) File "/home/kamran/.local/lib/python3.8/site-packages/mmcv/runner/runner.py", line 359, in run epoch_runner(data_loaders[i], kwargs) File "/home/kamran/.local/lib/python3.8/site-packages/mmcv/runner/runner.py", line 262, in train outputs = self.batch_processor( File "/home/kamran/workspace/mmdetection/mmdet/apis/train.py", line 75, in batch_processor losses = model(data) File "/usr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in call result = self.forward(*input, kwargs) File "/usr/lib/python3.8/site-packages/torch/nn/parallel/data_parallel.py", line 150, in forward return self.module(*inputs[0], *kwargs[0]) File "/usr/lib/python3.8/site-packages/torch/nn/modules/module.py", line 532, in call result = self.forward(input, kwargs) File "/home/kamran/workspace/mmdetection/mmdet/core/fp16/decorators.py", line 49, in new_func return old_func(args, kwargs) File "/home/kamran/workspace/mmdetection/mmdet/models/detectors/base.py", line 147, in forward return self.forward_train(img, img_metas, kwargs) File "/home/kamran/workspace/mmdetection/mmdet/models/detectors/two_stage.py", line 182, in forward_train proposal_list = self.rpn_head.get_bboxes(proposal_inputs) File "/home/kamran/workspace/mmdetection/mmdet/core/fp16/decorators.py", line 127, in new_func return old_func(*args, **kwargs) File "/home/kamran/workspace/mmdetection/mmdet/models/anchor_heads/anchor_head.py", line 274, in get_bboxes proposals = self.get_bboxes_single(cls_score_list, bbox_pred_list, File "/home/kamran/workspace/mmdetection/mmdet/models/anchor_heads/rpn_head.py", line 92, in get_bboxessingle proposals, = nms(proposals, cfg.nms_thr) File "/home/kamran/workspace/mmdetection/mmdet/ops/nms/nms_wrapper.py", line 54, in nms inds = nms_cuda.nms(dets_th, iou_thr) RuntimeError: !dets.type().is_cuda() INTERNAL ASSERT FAILED at mmdet/ops/nms/src/nms_cpu.cpp:7, please report a bug to PyTorch. dets must be a CPU tensor (nms_cpu_kernel at mmdet/ops/nms/src/nms_cpu.cpp:7) frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&) + 0x74 (0x7f637ab64734 in /usr/lib/python3.8/site-packages/torch/lib/libc10.so) frame #1: at::Tensor nms_cpu_kernel(at::Tensor const&, float) + 0xa2c (0x7f6315af683c in /home/kamran/workspace/mmdetection/mmdet/ops/nms/nms_cpu.cpython-38-x86_64-linux-gnu.so) frame #2: nms(at::Tensor const&, float) + 0x6d (0x7f6315ae1b2d in /home/kamran/workspace/mmdetection/mmdet/ops/nms/nms_cpu.cpython-38-x86_64-linux-gnu.so) frame #3: + 0x3b81d (0x7f6315aa881d in /home/kamran/workspace/mmdetection/mmdet/ops/nms/nms_cuda.cpython-38-x86_64-linux-gnu.so) frame #4: + 0x3913f (0x7f6315aa613f in /home/kamran/workspace/mmdetection/mmdet/ops/nms/nms_cuda.cpython-38-x86_64-linux-gnu.so)

Thank you

open-mmlab / mmdetection

RuntimeError in nms_cuda #2412