msracver / Deformable-ConvNets

Deformable Convolutional Networks
MIT License
4.04k stars 959 forks source link

raise MXNetError(py_str(_LIB.MXGetLastError())) #259

Open wander1985 opened 5 years ago

wander1985 commented 5 years ago

Got this error when running "python experiments/rfcn/rfcn_end2end_train_test.py --cfg experiments/rfcn/cfgs/user_resnet_v1_101_voc0712_rfcn_end2end_ohem.yaml" The error happened when the rfcn is doing testing.

experiments/rfcn....\rfcn\config\config.py:175: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details. exp_config = edict(yaml.load(f)) ('Called with argument:', Namespace(cfg='experiments/rfcn/cfgs/user_resnet_v1_101_voc0712_rfcn_end2end_ohem.yaml', frequent=100)) {'CLASS_AGNOSTIC': True, 'MXNET_VERSION': 'mxnet', 'SCALES': [(600, 1000)], 'TEST': {'BATCH_IMAGES': 1, 'CXX_PROPOSAL': False, 'HAS_RPN': True, 'NMS': 0.3, 'PROPOSAL_MIN_SIZE': 0, 'PROPOSAL_NMS_THRESH': 0.7, 'PROPOSAL_POST_NMS_TOP_N': 2000, 'PROPOSAL_PRE_NMS_TOP_N': 20000, 'RPN_MIN_SIZE': 0, 'RPN_NMS_THRESH': 0.7, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'max_per_image': 300, 'test_epoch': 7}, 'TRAIN': {'ALTERNATE': {'RCNN_BATCH_IMAGES': 0, 'RPN_BATCH_IMAGES': 0, 'rfcn1_epoch': 0, 'rfcn1_lr': 0, 'rfcn1_lr_step': '', 'rfcn2_epoch': 0, 'rfcn2_lr': 0, 'rfcn2_lr_step': '', 'rpn1_epoch': 0, 'rpn1_lr': 0, 'rpn1_lr_step': '', 'rpn2_epoch': 0, 'rpn2_lr': 0, 'rpn2_lr_step': '', 'rpn3_epoch': 0, 'rpn3_lr': 0, 'rpn3_lr_step': ''}, 'ASPECT_GROUPING': True, 'BATCH_IMAGES': 1, 'BATCH_ROIS': -1, 'BATCH_ROIS_OHEM': 128, 'BBOX_MEANS': [0.0, 0.0, 0.0, 0.0], 'BBOX_NORMALIZATION_PRECOMPUTED': True, 'BBOX_REGRESSION_THRESH': 0.5, 'BBOX_STDS': [0.1, 0.1, 0.2, 0.2], 'BBOX_WEIGHTS': array([1., 1., 1., 1.]), 'BG_THRESH_HI': 0.5, 'BG_THRESH_LO': 0.0, 'CXX_PROPOSAL': False, 'ENABLE_OHEM': True, 'END2END': True, 'FG_FRACTION': 0.25, 'FG_THRESH': 0.5, 'FLIP': False, 'RESUME': False, 'RPN_BATCH_SIZE': 256, 'RPN_BBOX_WEIGHTS': [1.0, 1.0, 1.0, 1.0], 'RPN_CLOBBER_POSITIVES': False, 'RPN_FG_FRACTION': 0.5, 'RPN_MIN_SIZE': 0, 'RPN_NEGATIVE_OVERLAP': 0.3, 'RPN_NMS_THRESH': 0.7, 'RPN_POSITIVE_OVERLAP': 0.7, 'RPN_POSITIVE_WEIGHT': -1.0, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'SHUFFLE': True, 'begin_epoch': 0, 'end_epoch': 1, 'lr': 5e-05, 'lr_factor': 0.1, 'lr_step': '4.83', 'model_prefix': 'rfcn_voc', 'momentum': 0.9, 'warmup': True, 'warmup_lr': 5e-06, 'warmup_step': 1000, 'wd': 0.0005}, 'dataset': {'NUM_CLASSES': 26, 'dataset': 'PascalVOC', 'dataset_path': './data/VOCdevkit', 'image_set': '2007_trainval', 'proposal': 'rpn', 'root_path': './data', 'test_image_set': '2007_test'}, 'default': {'frequent': 100, 'kvstore': 'device'}, 'gpus': '0', 'network': {'ANCHOR_RATIOS': [0.5, 1, 2], 'ANCHOR_SCALES': [8, 16, 32], 'FIXED_PARAMS': ['conv1', 'bn_conv1', 'res2', 'bn2', 'gamma', 'beta'], 'FIXED_PARAMS_SHARED': ['conv1', 'bn_conv1', 'res2', 'bn2', 'res3', 'bn3', 'res4', 'bn4', 'gamma', 'beta'], 'IMAGE_STRIDE': 0, 'NUM_ANCHORS': 9, 'PIXEL_MEANS': array([103.06, 115.9 , 123.15]), 'RCNN_FEAT_STRIDE': 16, 'RPN_FEAT_STRIDE': 16, 'pretrained': './model/pretrained_model/resnet_v1_101', 'pretrained_epoch': 0}, 'output_path': './output/rfcn/voc', 'symbol': 'resnet_v1_101_rfcn'} num_images 7325 voc_2007_trainval gt roidb loaded from ./data\cache\voc_2007_trainval_gt_roidb.pkl filtered 0 roidb entries: 7325 -> 7325 ('providing maximum shape', [('data', (1, 3, 600, 1000)), ('gt_boxes', (1, 100, 5))], [('label', (1, 21546L)), ('bbox_target', (1, 36L, 38L, 63L)), ('bbox_weight', (1, 36L, 38L, 63L))]) {'bbox_target': (1L, 36L, 38L, 38L), 'bbox_weight': (1L, 36L, 38L, 38L), 'data': (1L, 3L, 600L, 600L), 'gt_boxes': (1L, 20L, 5L), 'im_info': (1L, 3L), 'label': (1L, 12996L)} ('lr', 5e-05, 'lr_epoch_diff', [4.83], 'lr_iters', [35379]) experiments/rfcn....\rfcn..\lib\bbox\bbox_transform.py:95: RuntimeWarning: invalid value encountered in log targets_dw = np.log(gt_widths / ex_widths) experiments/rfcn....\rfcn..\lib\bbox\bbox_transform.py:96: RuntimeWarning: invalid value encountered in log targets_dh = np.log(gt_heights / ex_heights) Epoch[0] Batch [100] Speed: 2.40 samples/sec Train-RPNAcc=0.703047, RPNLogLoss=0.610697, RPNL1Loss=1.447503, RCNNAcc=0.012067, RCNNLogLoss=3.261478, RCNNL1Loss=0.008930, experiments/rfcn....\rfcn\operator_py\proposal.py:175: RuntimeWarning: invalid value encountered in greater_equal keep = np.where((ws >= min_size) & (hs >= min_size))[0] Epoch[0] Batch [200] Speed: 2.47 samples/sec Train-RPNAcc=0.736234, RPNLogLoss=0.610095, RPNL1Loss=nan, RCNNAcc=0.067708, RCNNLogLoss=3.254745, RCNNL1Loss=0.007116, ... Epoch[0] Batch [7300] Speed: 2.24 samples/sec Train-RPNAcc=0.764898, RPNLogLoss=0.650768, RPNL1Loss=nan, RCNNAcc=0.818957, RCNNLogLoss=1.028976, RCNNL1Loss=0.012867,
Namespace(cfg='experiments/rfcn/cfgs/user_resnet_v1_101_voc0712_rfcn_end2end_ohem.yaml', ignore_cache=False, shuffle=False, thresh=0.001, vis=False) {'CLASS_AGNOSTIC': True, 'MXNET_VERSION': 'mxnet', 'SCALES': [(600, 1000)], 'TEST': {'BATCH_IMAGES': 1, 'CXX_PROPOSAL': False, 'HAS_RPN': True, 'NMS': 0.3, 'PROPOSAL_MIN_SIZE': 0, 'PROPOSAL_NMS_THRESH': 0.7, 'PROPOSAL_POST_NMS_TOP_N': 2000, 'PROPOSAL_PRE_NMS_TOP_N': 20000, 'RPN_MIN_SIZE': 0, 'RPN_NMS_THRESH': 0.7, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'max_per_image': 300, 'test_epoch': 7}, 'TRAIN': {'ALTERNATE': {'RCNN_BATCH_IMAGES': 0, 'RPN_BATCH_IMAGES': 0, 'rfcn1_epoch': 0, 'rfcn1_lr': 0, 'rfcn1_lr_step': '', 'rfcn2_epoch': 0, 'rfcn2_lr': 0, 'rfcn2_lr_step': '', 'rpn1_epoch': 0, 'rpn1_lr': 0, 'rpn1_lr_step': '', 'rpn2_epoch': 0, 'rpn2_lr': 0, 'rpn2_lr_step': '', 'rpn3_epoch': 0, 'rpn3_lr': 0, 'rpn3_lr_step': ''}, 'ASPECT_GROUPING': True, 'BATCH_IMAGES': 1, 'BATCH_ROIS': -1, 'BATCH_ROIS_OHEM': 128, 'BBOX_MEANS': [0.0, 0.0, 0.0, 0.0], 'BBOX_NORMALIZATION_PRECOMPUTED': True, 'BBOX_REGRESSION_THRESH': 0.5, 'BBOX_STDS': [0.1, 0.1, 0.2, 0.2], 'BBOX_WEIGHTS': array([1., 1., 1., 1.]), 'BG_THRESH_HI': 0.5, 'BG_THRESH_LO': 0.0, 'CXX_PROPOSAL': False, 'ENABLE_OHEM': True, 'END2END': True, 'FG_FRACTION': 0.25, 'FG_THRESH': 0.5, 'FLIP': False, 'RESUME': False, 'RPN_BATCH_SIZE': 256, 'RPN_BBOX_WEIGHTS': [1.0, 1.0, 1.0, 1.0], 'RPN_CLOBBER_POSITIVES': False, 'RPN_FG_FRACTION': 0.5, 'RPN_MIN_SIZE': 0, 'RPN_NEGATIVE_OVERLAP': 0.3, 'RPN_NMS_THRESH': 0.7, 'RPN_POSITIVE_OVERLAP': 0.7, 'RPN_POSITIVE_WEIGHT': -1.0, 'RPN_POST_NMS_TOP_N': 300, 'RPN_PRE_NMS_TOP_N': 6000, 'SHUFFLE': True, 'begin_epoch': 0, 'end_epoch': 1, 'lr': 5e-05, 'lr_factor': 0.1, 'lr_step': '4.83', 'model_prefix': 'rfcn_voc', 'momentum': 0.9, 'warmup': True, 'warmup_lr': 5e-06, 'warmup_step': 1000, 'wd': 0.0005}, 'dataset': {'NUM_CLASSES': 26, 'dataset': 'PascalVOC', 'dataset_path': './data/VOCdevkit', 'image_set': '2007_trainval', 'proposal': 'rpn', 'root_path': './data', 'test_image_set': '2007_test'}, 'default': {'frequent': 100, 'kvstore': 'device'}, 'gpus': '0', 'network': {'ANCHOR_RATIOS': [0.5, 1, 2], 'ANCHOR_SCALES': [8, 16, 32], 'FIXED_PARAMS': ['conv1', 'bn_conv1', 'res2', 'bn2', 'gamma', 'beta'], 'FIXED_PARAMS_SHARED': ['conv1', 'bn_conv1', 'res2', 'bn2', 'res3', 'bn3', 'res4', 'bn4', 'gamma', 'beta'], 'IMAGE_STRIDE': 0, 'NUM_ANCHORS': 9, 'PIXEL_MEANS': array([103.06, 115.9 , 123.15]), 'RCNN_FEAT_STRIDE': 16, 'RPN_FEAT_STRIDE': 16, 'pretrained': './model/pretrained_model/resnet_v1_101', 'pretrained_epoch': 0}, 'output_path': './output/rfcn/voc', 'symbol': 'resnet_v1_101_rfcn'} num_images 245 wrote gt roidb to ./data\cache\voc_2007_test_gt_roidb.pkl Traceback (most recent call last): File "experiments/rfcn/rfcn_end2end_train_test.py", line 20, in test.main() File "experiments/rfcn....\rfcn\test.py", line 57, in main args.vis, args.ignore_cache, args.shuffle, config.TEST.HAS_RPN, config.dataset.proposal, args.thresh, logger=logger, output_path=final_output_path) File "experiments/rfcn....\rfcn\function\test_rcnn.py", line 56, in test_rcnn arg_params, aux_params = load_param(prefix, epoch, process=True) File "experiments/rfcn....\rfcn..\lib\utils\load_model.py", line 56, in load_param arg_params, aux_params = load_checkpoint(prefix, epoch) File "experiments/rfcn....\rfcn..\lib\utils\load_model.py", line 22, in load_checkpoint save_dict = mx.nd.load('%s-%04d.params' % (prefix, epoch)) File "d:\Anaconda2\envs\dcn\lib\site-packages\mxnet\ndarray\utils.py", line 175, in load ctypes.byref(names))) File "d:\Anaconda2\envs\dcn\lib\site-packages\mxnet\base.py", line 146, in check_call raise MXNetError(py_str(_LIB.MXGetLastError())) mxnet.base.MXNetError: [17:30:52] C:\Jenkins\workspace\mxnet-tag\mxnet\dmlc-core\src\io\local_filesys.cc:166: Check failed: allow_null LocalFileSystem: fail to open "./output/rfcn/voc\user_resnet_v1_101_voc0712_rfcn_end2end_ohem\2007_test..\2007_trainval\rfcn_voc-0007.params"

Please help. Thanks in advance.