I want to train the DOTA in specific image scale

Hello,thanks for your work! I have the question that I want to train the DOTA dataset in specific image-scale like 768*768 and some classes like plane or storage tank which is splitted by the DOTA-devkit. But I got the training problem like below: ('Called with argument:', Namespace(cfg='experiments/faster_rcnn/cfgs/DOTA_quadrangle.yaml', frequent=100)) {'CLASS_AGNOSTIC': False, 'MXNET_VERSION': 'mxnet', 'RESIZE_TO_FIX_SIZE': True, 'SCALES': [(768, 768)], 'TEST': {'BATCH_IMAGES': 1, 'CXX_PROPOSAL': False, 'DO_MULTISCALE_TEST': False, 'HAS_RPN': True, 'MULTISCALE': [1.0, 1.2, 1.4, 1.6], 'NMS': 0.3, 'PROPOSAL_MIN_SIZE': 0, 'PROPOSAL_NMS_THRESH': 0.7, 'PROPOSAL_POST_NMS_TOP_N': 2000, 'PROPOSAL_PRE_NMS_TOP_N': 20000, 'RPN_MIN_SIZE': 0, 'RPN_NMS_THRESH': 0.7, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'max_per_image': 300, 'save_img_path': '/home/zxr/houweining/test_frcnn/Faster_RCNN_for_DOTA/data/hwn/model_01_vis', 'test_epoch': 59}, 'TRAIN': {'ALTERNATE': {'RCNN_BATCH_IMAGES': 0, 'RPN_BATCH_IMAGES': 0, 'rfcn1_epoch': 0, 'rfcn1_lr': 0, 'rfcn1_lr_step': '', 'rfcn2_epoch': 0, 'rfcn2_lr': 0, 'rfcn2_lr_step': '', 'rpn1_epoch': 0, 'rpn1_lr': 0, 'rpn1_lr_step': '', 'rpn2_epoch': 0, 'rpn2_lr': 0, 'rpn2_lr_step': '', 'rpn3_epoch': 0, 'rpn3_lr': 0, 'rpn3_lr_step': ''}, 'ASPECT_GROUPING': True, 'BATCH_IMAGES': 1, 'BATCH_ROIS': 128, 'BATCH_ROIS_OHEM': 128, 'BBOX_MEANS': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'BBOX_NORMALIZATION_PRECOMPUTED': False, 'BBOX_REGRESSION_THRESH': 0.5, 'BBOX_STDS': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'BBOX_WEIGHTS': array([1., 1., 1., 1., 1., 1., 1., 1.]), 'BG_THRESH_HI': 0.5, 'BG_THRESH_LO': 0.1, 'CXX_PROPOSAL': False, 'ENABLE_OHEM': True, 'END2END': True, 'FG_FRACTION': 0.25, 'FG_THRESH': 0.5, 'FLIP': True, 'RESUME': False, 'RPN_BATCH_SIZE': 256, 'RPN_BBOX_WEIGHTS': [1.0, 1.0, 1.0, 1.0], 'RPN_CLOBBER_POSITIVES': False, 'RPN_FG_FRACTION': 0.5, 'RPN_MIN_SIZE': 0, 'RPN_NEGATIVE_OVERLAP': 0.3, 'RPN_NMS_THRESH': 0.7, 'RPN_POSITIVE_OVERLAP': 0.7, 'RPN_POSITIVE_WEIGHT': -1.0, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'SHUFFLE': True, 'begin_epoch': 0, 'end_epoch': 60, 'lr': 0.0005, 'lr_factor': 0.1, 'lr_step': '45,52', 'model_prefix': 'rcnn_DOTA_quadrangle', 'momentum': 0.9, 'warmup': True, 'warmup_lr': 5e-05, 'warmup_step': 1000, 'wd': 0.0005}, 'dataset': {'NUM_CLASSES': 2, 'dataset': 'DOTA_oriented', 'dataset_path': '/home/zxr/houweining/test_frcnn/Faster_RCNN_for_DOTA/data/dota_for_obb_768', 'image_set': 'train', 'proposal': 'rpn', 'root_path': '/home/zxr/houweining/test_frcnn/Faster_RCNN_for_DOTA/data/hwn/model_01', 'test_image_set': 'test'}, 'default': {'frequent': 100, 'kvstore': 'device'}, 'gpus': '0', 'network': {'ANCHOR_RATIOS': [0.5, 1, 2], 'ANCHOR_SCALES': [8, 16, 32], 'FIXED_PARAMS': ['conv1', 'bn_conv1', 'res2', 'bn2', 'gamma', 'beta'], 'FIXED_PARAMS_SHARED': ['conv1', 'bn_conv1', 'res2', 'bn2', 'res3', 'bn3', 'res4', 'bn4', 'gamma', 'beta'], 'IMAGE_STRIDE': 0, 'NUM_ANCHORS': 9, 'PIXEL_MEANS': array([103.06, 115.9 , 123.15]), 'RCNN_FEAT_STRIDE': 16, 'RPN_FEAT_STRIDE': 16, 'pretrained': './model/pretrained_model/resnet_v1_101', 'pretrained_epoch': 0}, 'output_path': './output/rcnn/DOTA_quadrangle', 'symbol': 'resnet_v1_101_rcnn_quadrangle'} num_images 2026 wrote gt roidb to /home/zxr/houweining/test_frcnn/Faster_RCNN_for_DOTA/data/hwn/model_01/cache/DOTA_oriented_train_gt_roidb.pkl append flipped images to roidb filtered 150 roidb entries: 4052 -> 3902 providing maximum shape [('data', (1, 3, 768, 768)), ('gt_boxes', (1, 100, 9))] [('label', (1, 20736)), ('bbox_target', (1, 36, 48, 48)), ('bbox_weight', (1, 36, 48, 48))] {'bbox_target': (1L, 36L, 48L, 48L), 'bbox_weight': (1L, 36L, 48L, 48L), 'data': (1L, 3L, 768L, 768L), 'gt_boxes': (1L, 3L, 9L), 'im_info': (1L, 3L), 'label': (1L, 20736L)} ('lr', 0.0005, 'lr_epoch_diff', [45.0, 52.0], 'lr_iters', [175590, 202904]) [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. [11:55:59] src/operator/convolution.cu:87: This convolution is not supported by cudnn, MXNET convolution is applied. Error in CustomOp.forward: Traceback (most recent call last): File "/home/zxr/anaconda3/envs/hwn_frcnn/lib/python2.7/site-packages/mxnet/operator.py", line 758, in forward_entry aux=tensors[4]) File "experiments/faster_rcnn/../../faster_rcnn/operator_py/proposal_target_quadrangle.py", line 87, in forward sample_rois_quadrangle(all_rois, fg_rois_per_image, rois_per_image, self._num_classes, self._cfg, gt_boxes=gt_boxes) File "experiments/faster_rcnn/../../faster_rcnn/core/rcnn.py", line 265, in sample_rois_quadrangle expand_bbox_regression_targets_quadrangle(bbox_target_data, num_classes, cfg) File "experiments/faster_rcnn/../../faster_rcnn/../lib/bbox/bbox_regression.py", line 157, in expand_bbox_regression_targets_quadrangle bbox_targets[index, start:end] = bbox_targets_data[index, 1:] ValueError: could not broadcast input array from shape (8) into shape (0)

[11:55:59] /home/ubuntu/mxnet-distro/mxnet-build/dmlc-core/include/dmlc/logging.h:304: [11:55:59] src/operator/custom/custom.cc:77: Check failed: reinterpret_cast(opinfo->callbacks[kCustomOpForward])( ptrs.size(), ptrs.data(), tags.data(), reqs.data(), static_cast(ctx.is_train), opinfo->contexts[kCustomOpForward])

Stack trace returned 6 entries: [bt] (0) /home/zxr/anaconda3/envs/hwn_frcnn/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x184dfc) [0x7fd31db3fdfc] [bt] (1) /home/zxr/anaconda3/envs/hwn_frcnn/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x248ec8) [0x7fd31dc03ec8] [bt] (2) /home/zxr/anaconda3/envs/hwn_frcnn/lib/python2.7/site-packages/mxnet/libmxnet.so(+0x2424ce) [0x7fd31dbfd4ce] [bt] (3) /home/zxr/anaconda3/envs/hwn_frcnn/bin/../lib/libstdc++.so.6(+0xb7260) [0x7fd346407260] [bt] (4) /lib/x86_64-linux-gnu/libpthread.so.0(+0x76ba) [0x7fd34d0576ba] [bt] (5) /lib/x86_64-linux-gnu/libc.so.6(clone+0x6d) [0x7fd34c67d41d]

terminate called after throwing an instance of 'dmlc::Error' what(): [11:55:59] src/operator/custom/custom.cc:77: Check failed: reinterpret_cast(opinfo->callbacks[kCustomOpForward])( ptrs.size(), ptrs.data(), tags.data(), reqs.data(), static_cast(ctx.is_train), opinfo->contexts[kCustomOpForward])

Aborted (core dumped)

I have checked my dataset that generated by DOTA-devkit but can't found the reason, can you give me some help or tell me which configures should be modified?

jessemelpolio / Faster_RCNN_for_DOTA

I want to train the DOTA in specific image scale #27