sjtuytc
commented
2 years ago I meet the same issue!
Open Dream-3000 opened 2 years ago
sjtuytc
commented
2 years ago I meet the same issue!
sjtuytc
commented
2 years ago workaround: close fp16
Luckygyana
commented
2 years ago workaround: close fp16
@sjtuytc , Can you describe how you solved this problem. I am using colab to use this, but couldn’t run till yet.
sjtuytc
commented
2 years ago workaround: close fp16
@sjtuytc , Can you describe how you solved this problem. I am using colab to use this, but couldn’t run till yet.
wliebtzy
commented
2 years ago @Dream-3000 Hi, have you solved the problem? Best Regards!
Luckygyana
commented
2 years ago @Dream-3000 Hi, have you solved the problem? Best Regards!
https://github.com/NVIDIA/apex/blob/0e25fcc433bbbe2c210eca0590827bdbf2a06960/apex/amp/utils.py#L97 in this line change [1][0] to [0][0]
qingtian5
commented
2 years ago @Dream-3000 Hi, have you solved the problem? Best Regards!
https://github.com/NVIDIA/apex/blob/0e25fcc433bbbe2c210eca0590827bdbf2a06960/apex/amp/utils.py#L97 in this line change [1][0] to [0][0]
solve my problem!thank u
2021-11-22 11:43:22,593 - mmdet - INFO - workflow: [('train', 1)], max: 12 epochs 2021-11-22 11:43:22,597 - mmdet - INFO - Checkpoints will be saved to /content/drive/My Drive/Swin-Transformer-Object-Detection/work_dirs/mask_rcnn_swin_tiny_patch4_window7_mstrain_480-800_adamw_1x_coco by HardDiskBackend. loading annotations into memory... Done (t=0.01s) creating index... index created! Selected optimization level O1: Insert automatic casts around Pytorch functions and Tensor methods.
Defaults for this optimization level are: enabled : True opt_level : O1 cast_model_type : None patch_torch_functions : True keep_batchnorm_fp32 : None master_weights : None loss_scale : dynamic Processing user overrides (additional kwargs that are not None)... After processing overrides, optimization options are: enabled : True opt_level : O1 cast_model_type : None patch_torch_functions : True keep_batchnorm_fp32 : None master_weights : None loss_scale : dynamic loading annotations into memory... Done (t=0.00s) creating index... index created!
IndexError Traceback (most recent call last) /content/drive/My Drive/Swin-Transformer-Object-Detection/tools/train.py in()
185
186 if name == 'main':
--> 187 main()
20 frames /content/drive/My Drive/Swin-Transformer-Object-Detection/tools/train.py in main() 181 validate=(not args.no_validate), 182 timestamp=timestamp, --> 183 meta=meta) 184 185
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/apis/train.py in train_detector(model, dataset, cfg, distributed, validate, timestamp, meta) 181 elif cfg.load_from: 182 runner.load_checkpoint(cfg.load_from) --> 183 runner.run(data_loaders, cfg.workflow)
/usr/local/lib/python3.7/dist-packages/mmcv/runner/epoch_based_runner.py in run(self, data_loaders, workflow, max_epochs, kwargs) 125 if mode == 'train' and self.epoch >= self._max_epochs: 126 break --> 127 epoch_runner(data_loaders[i], kwargs) 128 129 time.sleep(1) # wait for some hooks like loggers to finish
/usr/local/lib/python3.7/dist-packages/mmcv/runner/epoch_based_runner.py in train(self, data_loader, kwargs) 48 self._inner_iter = i 49 self.call_hook('before_train_iter') ---> 50 self.run_iter(data_batch, train_mode=True, kwargs) 51 self.call_hook('after_train_iter') 52 self._iter += 1
/usr/local/lib/python3.7/dist-packages/mmcv/runner/epoch_based_runner.py in run_iter(self, data_batch, train_mode, kwargs) 28 elif train_mode: 29 outputs = self.model.train_step(data_batch, self.optimizer, ---> 30 kwargs) 31 else: 32 outputs = self.model.val_step(data_batch, self.optimizer, **kwargs)
/usr/local/lib/python3.7/dist-packages/mmcv/parallel/data_parallel.py in train_step(self, *inputs, kwargs) 65 66 inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) ---> 67 return self.module.train_step(*inputs[0], *kwargs[0]) 68 69 def val_step(self, inputs, kwargs):
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/detectors/base.py in train_step(self, data, optimizer) 245 averaging the logs. 246 """ --> 247 losses = self(**data) 248 loss, log_vars = self._parse_losses(losses) 249
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, *kwargs) 1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks 1101 or _global_forward_hooks or _global_forward_pre_hooks): -> 1102 return forward_call(input, **kwargs) 1103 # Do not call functions when jit is used 1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/mmcv/runner/fp16_utils.py in new_func(*args, kwargs) 126 digit_version(TORCH_VERSION) >= digit_version('1.6.0')): 127 with autocast(enabled=True): --> 128 output = old_func(*new_args, *new_kwargs) 129 else: 130 output = old_func(new_args, new_kwargs)
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/detectors/base.py in forward(self, img, img_metas, return_loss, kwargs) 179 """ 180 if return_loss: --> 181 return self.forward_train(img, img_metas, kwargs) 182 else: 183 return self.forward_test(img, img_metas, **kwargs)
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/detectors/two_stage.py in forward_train(self, img, img_metas, gt_bboxes, gt_labels, gt_bboxes_ignore, gt_masks, proposals, **kwargs) 154 gt_labels=None, 155 gt_bboxes_ignore=gt_bboxes_ignore, --> 156 proposal_cfg=proposal_cfg) 157 losses.update(rpn_losses) 158 else:
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/dense_heads/base_dense_head.py in forward_train(self, x, img_metas, gt_bboxes, gt_labels, gt_bboxes_ignore, proposal_cfg, **kwargs) 47 proposal_list (list[Tensor]): Proposals of each image. 48 """ ---> 49 outs = self(x) 50 if gt_labels is None: 51 loss_inputs = outs + (gt_bboxes, img_metas)
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, *kwargs) 1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks 1101 or _global_forward_hooks or _global_forward_pre_hooks): -> 1102 return forward_call(input, **kwargs) 1103 # Do not call functions when jit is used 1104 full_backward_hooks, non_full_backward_hooks = [], []
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/dense_heads/anchor_head.py in forward(self, feats) 141 is num_anchors * 4. 142 """ --> 143 return multi_apply(self.forward_single, feats) 144 145 def get_anchors(self, featmap_sizes, img_metas, device='cuda'):
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/core/utils/misc.py in multi_apply(func, args, kwargs) 27 pfunc = partial(func, kwargs) if kwargs else func 28 map_results = map(pfunc, args) ---> 29 return tuple(map(list, zip(*map_results))) 30 31
/content/drive/MyDrive/Swin-Transformer-Object-Detection/mmdet/models/dense_heads/rpn_head.py in forward_single(self, x) 41 def forward_single(self, x): 42 """Forward feature map of a single scale level.""" ---> 43 x = self.rpn_conv(x) 44 x = F.relu(x, inplace=True) 45 rpn_cls_score = self.rpn_cls(x)
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, *kwargs) 1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks 1101 or _global_forward_hooks or _global_forward_pre_hooks): -> 1102 return forward_call(input, **kwargs) 1103 # Do not call functions when jit is used 1104 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in forward(self, input) 444 445 def forward(self, input: Tensor) -> Tensor: --> 446 return self._conv_forward(input, self.weight, self.bias) 447 448 class Conv3d(_ConvNd):
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight, bias) 441 _pair(0), self.dilation, self.groups) 442 return F.conv2d(input, weight, bias, self.stride, --> 443 self.padding, self.dilation, self.groups) 444 445 def forward(self, input: Tensor) -> Tensor:
/usr/local/lib/python3.7/dist-packages/apex-0.1-py3.7-linux-x86_64.egg/apex/amp/wrap.py in wrapper(*args, **kwargs) 19 for i in range(len(args)): 20 if utils.should_cache(args[i]): ---> 21 args[i] = utils.cached_cast(cast_fn, args[i], handle.cache) 22 for k in kwargs: 23 if utils.should_cache(kwargs[k]):
/usr/local/lib/python3.7/dist-packages/apex-0.1-py3.7-linux-x86_64.egg/apex/amp/utils.py in cached_cast(cast_fn, x, cache) 95 if x.requires_grad and cached_x.requires_grad: 96 # Make sure x is actually cached_x's autograd parent. ---> 97 if cached_x.grad_fn.next_functions[1][0].variable is not x: 98 raise RuntimeError("x and cache[x] both require grad, but x is not " 99 "cache[x]'s parent. This is likely an error.")
IndexError: tuple index out of range