Error running model on c++

[InfiniteLife]

I converted model using dpt_scriptable branch, like this:

    model.eval()

    if optimize == True and device == torch.device("cuda"):
        model = torch.jit.script(model)
        model = model.to(memory_format=torch.channels_last)
        model = model.half()

    model.to(device)

    model.save(model_path[:-3] + ".torchscript.pt")

then I tried to using it in C++, loading Mat image and converting it to PyTorch:

    cv::Mat ch_first = data.clone();

    if (data.type() != CV_32FC3) cout << "wrong type" << endl;

    float* feed_data = (float*)data.data;
    float* ch_first_data = (float*)ch_first.data;

    for (int p = 0; p < (int)data.total(); ++p)
    {
        // R
        ch_first_data[p] = feed_data[p * 3];
        // G
        ch_first_data[p + (int)data.total()] = feed_data[p * 3 + 1];
        // B
        ch_first_data[p + 2 * (int)data.total()] = feed_data[p * 3 + 2];
    }

    torch::Tensor image_input = torch::from_blob((float*)ch_first.data, { 1, data.rows, data.cols, 3 });
    image_input = image_input.toType(torch::kFloat16);

    image_input = image_input.to((*device));

    auto net_out = module.forward({ image_input });

data height is 384 and width is 672. In for Im just unpacking values from OpenCV byte order to pytorch byte order.

And in forward function I recieve exception:

    KernelBase.dll!00007ffcea784f99()   Unknown
    vcruntime140d.dll!00007ffc5afab460()    Unknown
>   torch_cpu.dll!torch::jit::InterpreterStateImpl::handleError(const torch::jit::ExceptionMessage & msg, bool is_jit_exception, c10::NotImplementedError * not_implemented_error) Line 665 C++
    torch_cpu.dll!`torch::jit::InterpreterStateImpl::runImpl'::`1'::catch$81() Line 639 C++
    [External Code] 
    torch_cpu.dll!torch::jit::InterpreterStateImpl::runImpl(std::vector<c10::IValue,std::allocator<c10::IValue>> & stack) Line 251  C++
    torch_cpu.dll!torch::jit::InterpreterStateImpl::run(std::vector<c10::IValue,std::allocator<c10::IValue>> & stack) Line 728  C++
    torch_cpu.dll!torch::jit::InterpreterState::run(std::vector<c10::IValue,std::allocator<c10::IValue>> & stack) Line 841  C++
    torch_cpu.dll!torch::jit::GraphExecutorImplBase::run(std::vector<c10::IValue,std::allocator<c10::IValue>> & stack) Line 544 C++
    torch_cpu.dll!torch::jit::GraphExecutor::run(std::vector<c10::IValue,std::allocator<c10::IValue>> & inputs) Line 767    C++
    torch_cpu.dll!torch::jit::GraphFunction::run(std::vector<c10::IValue,std::allocator<c10::IValue>> & stack) Line 36  C++
    torch_cpu.dll!torch::jit::GraphFunction::operator()(std::vector<c10::IValue,std::allocator<c10::IValue>> stack, const std::unordered_map<std::string,c10::IValue,std::hash<std::string>,std::equal_to<std::string>,std::allocator<std::pair<std::string const ,c10::IValue>>> & kwargs) Line 53 C++
    torch_cpu.dll!torch::jit::Method::operator()(std::vector<c10::IValue,std::allocator<c10::IValue>> stack, const std::unordered_map<std::string,c10::IValue,std::hash<std::string>,std::equal_to<std::string>,std::allocator<std::pair<std::string const ,c10::IValue>>> & kwargs) Line 225   C++
    torch_cpu.dll!torch::jit::Module::forward(std::vector<c10::IValue,std::allocator<c10::IValue>> inputs) Line 114 C++
    pytorch_test.exe!main() Line 128    C++

I connected source file for debugging and got exception string:

The following operation failed in the TorchScript interpreter.
Traceback of TorchScript, serialized code (most recent call last):
  File "code/__torch__/dpt/models.py", line 14, in forward
  def forward(self: __torch__.dpt.models.DPTDepthModel,
    x: Tensor) -> Tensor:
    inv_depth = torch.squeeze((self).forward_features(x, ), 1)
                               ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
    if self.invert:
      depth = torch.add(torch.mul(inv_depth, self.scale), self.shift)
  File "code/__torch__/dpt/models.py", line 28, in forward_features
  def forward_features(self: __torch__.dpt.models.DPTDepthModel,
    x: Tensor) -> Tensor:
    layer_1, layer_2, layer_3, layer_4, = (self.pretrained).forward(x, )
                                           ~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
    layer_1_rn = (self.scratch.layer1_rn).forward(layer_1, )
    layer_2_rn = (self.scratch.layer2_rn).forward(layer_2, )
  File "code/__torch__/dpt/vit.py", line 22, in forward
    x: Tensor) -> Tuple[Tensor, Tensor, Tensor, Tensor]:
    _0, _1, h, w, = torch.size(x)
    layers = (self).forward_flex(x, )
              ~~~~~~~~~~~~~~~~~~ <--- HERE
    layer_1, layer_2, layer_3, layer_4, = layers
    layer_10 = (self.readout_oper1).forward(layer_1, )
  File "code/__torch__/dpt/vit.py", line 54, in forward_flex
    _15 = torch.floordiv(H, (self.patch_size)[1])
    _16 = torch.floordiv(W, (self.patch_size)[0])
    pos_embed = (self)._resize_pos_embed(_14, _15, _16, )
                 ~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
    B0 = (torch.size(x))[0]
    _17 = (self.model.patch_embed.proj).forward(x, )
  File "code/__torch__/dpt/vit.py", line 220, in _resize_pos_embed
    _68 = torch.reshape(posemb_grid, [1, gs_old, gs_old, -1])
    posemb_grid0 = torch.permute(_68, [0, 3, 1, 2])
    posemb_grid1 = _67(posemb_grid0, [gs_h, gs_w], None, "bilinear", False, None, )
                   ~~~ <--- HERE
    _69 = torch.permute(posemb_grid1, [0, 2, 3, 1])
    posemb_grid2 = torch.reshape(_69, [1, torch.mul(gs_h, gs_w), -1])
  File "code/__torch__/torch/nn/functional/___torch_mangle_25.py", line 256, in interpolate
                    ops.prim.RaiseException("AssertionError: ")
                    align_corners6 = _25
                  _81 = torch.upsample_bilinear2d(input, output_size2, align_corners6, scale_factors5)
                        ~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
                  _79 = _81
                else:

Traceback of TorchScript, original code (most recent call last):
  File "D:\dev\Mars\DPT-dpt_scriptable\dpt\models.py", line 114, in forward
    def forward(self, x):
        inv_depth = self.forward_features(x).squeeze(dim=1)
                    ~~~~~~~~~~~~~~~~~~~~~ <--- HERE

        if self.invert:
  File "D:\dev\Mars\DPT-dpt_scriptable\dpt\vit.py", line 302, in forward
        _, _, h, w = x.shape

        layers = self.forward_flex(x)
                 ~~~~~~~~~~~~~~~~~ <--- HERE

        # HACK: this is to make TorchScript happy. Can't directly address modules,
  File "D:\dev\Mars\DPT-dpt_scriptable\dpt\vit.py", line 259, in forward_flex
        B, _, H, W = x.shape

        pos_embed = self._resize_pos_embed(
                    ~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
            self.model.pos_embed,
            int(H // self.patch_size[1]),
  File "D:\dev\Mars\DPT-dpt_scriptable\dpt\vit.py", line 247, in _resize_pos_embed

        posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2)
        posemb_grid = F.interpolate(
                      ~~~~~~~~~~~~~ <--- HERE
            posemb_grid, size=[gs_h, gs_w], mode="bilinear", align_corners=False
        )
  File "C:\Users\Ilya\anaconda3\envs\np\lib\site-packages\torch\nn\functional.py", line 3709, in interpolate
    if input.dim() == 4 and mode == "bilinear":
        assert align_corners is not None
        return torch._C._nn.upsample_bilinear2d(input, output_size, align_corners, scale_factors)
               ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE
    if input.dim() == 5 and mode == "trilinear":
        assert align_corners is not None
RuntimeError: Input and output sizes should be greater than 0, but got input (H: 24, W: 24) output (H: 42, W: 0)

It seems something wrong with input size? Should it be 384x384? I am not sure what is wrong

[ranftlr]

Any resolution that can evenly be divided by 32 works, so that should be fine in this case. Hard to tell what the issue is, but here is what I would try:

• Try setting channels_last=False before exporting and feed the input tensors in BCHW order.
• Try using full precision

[InfiniteLife]

Apparently I needed to convert model to half() after loading it. I assumed libtorch loads already halfed model, as saved model was half of weights of original model, but it is wrong. Now it works.

[Wing100]

@InfiniteLife Hello, did you convert the pt_hybrid-midas-d889a10e.pt model? Has your problem been solved? I have encountered a similar problem recently. Can I take a look at your C++ code?

[yohannes-taye]

I came across the same error and I just modified the order of the inputs from torch::Tensor image_input = torch::from_blob((float*)ch_first.data, { 1, data.rows, data.cols, 3 }); to torch::Tensor image_input = torch::from_blob((float*)ch_first.data, { 1, 3, data.rows, data.cols});