Open RiverLight4 opened 6 months ago
If I input img only:
with RewriterContext(**context_info), torch.no_grad():
# for exporting models with weight that depends on inputs
func(*inputs) if isinstance(inputs, Sequence) \
else func(return_loss=False, rescale=True, **inputs) # This works correct, **kwargs relays correctly and img_metas has ['ori_shape'] etc. during processing.
ts_model = torch.jit.trace(
func,
inputs['img'][0], # only input img, not "inputs['img_metas']" as torch.jit.trace input.
check_trace=check_trace,
check_tolerance=check_tolerance,
strict=False
)
results:
Process Process-2:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/usr/local/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/core/pipeline_manager.py", line 107, in __call__
ret = func(*args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/pytorch2torchscript_edit.py", line 65, in torch2torchscript_edit
check_trace=False)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/core/pipeline_manager.py", line 356, in _wrap
return self.call_function(func_name_, *args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/core/pipeline_manager.py", line 326, in call_function
return self.call_function_local(func_name, *args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/core/pipeline_manager.py", line 275, in call_function_local
return pipe_caller(*args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/core/pipeline_manager.py", line 107, in __call__
ret = func(*args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/apis/torch_jit/trace_edit.py", line 137, in trace_edit
strict=False
File "/workspaces/rm-test/.venv/lib/python3.7/site-packages/torch/jit/_trace.py", line 768, in trace
_module_class,
File "/workspaces/rm-test/.venv/lib/python3.7/site-packages/torch/jit/_trace.py", line 983, in trace_module
argument_names,
File "/workspaces/rm-test/.venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/workspaces/rm-test/.venv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 1182, in _slow_forward
result = self.forward(*input, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/core/rewriters/rewriter_utils.py", line 402, in wrapper
return self.func(self, *args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/codebase/mmdet/models/detectors/base.py", line 70, in base_detector__forward
return __forward_impl(ctx, self, img, img_metas=img_metas, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/core/optimizers/function_marker.py", line 261, in g
rets = f(*args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/codebase/mmdet/models/detectors/base.py", line 26, in __forward_impl
return self.simple_test(img, img_metas, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/core/rewriters/rewriter_utils.py", line 402, in wrapper
return self.func(self, *args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/codebase/mmdet/models/detectors/two_stage.py", line 59, in two_stage_detector__simple_test
return self.roi_head.simple_test(x, proposals, img_metas, rescale=True)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/core/rewriters/rewriter_utils.py", line 402, in wrapper
return self.func(self, *args, **kwargs)
File "/workspaces/rm-test/mmdeploy-0.14.0/mmdeploy/codebase/mmdet/models/roi_heads/standard_roi_head.py", line 59, in standard_roi_head__simple_test
x, img_metas, det_bboxes, det_labels, rescale=True)
File "/workspaces/rm-test/mmdetection-2.28.2/mmdet/models/roi_heads/refine_roi_head.py", line 86, in simple_test_mask
ori_shape = img_metas[0]['ori_shape']
Error model code:
at mmdetection-2.28.2/mmdet/models/roi_heads/refine_roi_head.py
(added by RefineMask)
def simple_test_mask(self, x, img_metas, det_bboxes, det_labels, rescale=False):
"""Simple test for mask head without augmentation."""
### TODO: WORKAROUND: Only 1st image is processed. It should be fixed to multiple image inputs
ori_shape = img_metas[0]['ori_shape'] ## ERROR AT HERE: img_metas does not have ['ori_shape'] because I couldn't input img_metas
scale_factor = img_metas[0]['scale_factor'] ## This statemante is also failed
if det_bboxes[0].shape[0] == 0:
segm_result = [[] for _ in range(self.mask_head.stage_num_classes[0])]
else:
Checklist
Describe the bug
Hello,
I'd like to use the model which is trained with RefineMask ( https://github.com/zhanggang001/RefineMask ) in ONNX. So I implemented RefineMask into MMDetection v2.28.2 and tried with MMDeploy v0.14.0, and tried converting the model to torchscript at first, but conversion is failed. By tracing MMDetection/MMDeploy code, I found that 'img_metas' is not forwarded when model is called by
torch.jit.trace
when converting. How could I give both 'img' and 'img_metas' throughtorch.jit.trace
?['img_metas'][0]['ori_shape']
and['img_metas'][0]['scale_factor']
are used in the model of RefineMask head, it is necessary to relayimg_metas
through the model.info
Required inputs:
Data generated with
LoadImageFromFile
like:e.g. input directly into
torch.jit.trace
using
mmdeploy-0.14.0/tools/deploy.py
at
mmdeploy-0.14.0/mmdeploy/apis/torch_jit/
Reproduction
git clone https://github.com/zhanggang001/RefineMask.git
git clone https://github.com/open-mmlab/mmdetection.git
cd mmdetection & git checkout v2.28.2 & cd ..
git clone https://github.com/open-mmlab/mmdeploy.git
cd mmdeploy & git checkout v0.14.0 & cd ..
add:
add & edit:
mmdet/models/roi_heads/refine_roi_head.py
see https://github.com/zhanggang001/RefineMask/issues/38edit:
Edit
mmdeploy/apis/pytorch2torchscript.py
Edit
mmdeploy/apis/torch_jit/trace.py
if changingtorch.jit.trace
inputrun model conversion with:
Environment
Error traceback