Error encountered while using libtorch for batch inference

Instructions To Reproduce the 🐛 Bug:

batch inference code:

int main() {

torch::jit::script::Module module = load_model();

auto device = (*begin(module.buffers())).device();
module.to(device);
module.eval();

torch::Tensor imgs_in = torch::empty({ 0 }).to(device);

string image_path = "D:/VsProjects/d2_deploy/test_pic";
vector<String> src_name;
glob(image_path, src_name);
int batchSize = 2;

for (int i = 0; i < src_name.size(); i++)
{

    cv::Mat input_img = cv::imread(src_name[i], cv::IMREAD_COLOR);
    resize(input_img, input_img, Size(500, 500));
    input_img.convertTo(input_img, CV_32FC3, 1.0f / 127.5f, -1);

    const int height = input_img.rows;
    const int width = input_img.cols;
    const int channels = 3;

    auto img_tensor =
        torch::from_blob(input_img.data, {1, height, width, channels }).to(device);
    cout << "img_tensor define" << endl;
    // HWC to CHW
    img_tensor =
        img_tensor.permute({0, 3, 1, 2 }).contiguous();

    imgs_in = torch::cat({ imgs_in,img_tensor }, 0);
    imgs_in.print();

    if ((i+1)%batchSize==0)
    {
        vector<c10::IValue> batchInputs;
        batchInputs.emplace_back(imgs_in);
        auto outputs = module.forward({imgs_in});
        batchInputs.clear();
    }
}
return 0;

}

2 . one picture inference code:

c10::IValue get_scripting_inputs(cv::Mat& img, c10::Device device) { const int height = img.rows; const int width = img.cols; const int channels = 3;

auto img_tensor = torch::from_blob(img.data, {height, width, channels}, torch::kUInt8); cout << "img_tensor define" << endl; // HWC to CHW img_tensor = img_tensor.to(device, torch::kFloat).permute({2, 0, 1}).contiguous(); cout << "------------------tensor to device:" << device << endl; c10::Dict dic = c10::Dict<std::string, torch::Tensor>(); dic.insert("image", img_tensor); cout << "------------get_scripting_inputs finish------" << endl; return std::make_tuple(dic); } int main() {

torch::jit::script::Module module = load_model();

auto device = (*begin(module.buffers())).device();

string image_path = "D:/VsProjects/d2_deploy/test_pic"; vector src_name; glob(image_path, src_name);

for (int i = 0; i <src_name.size(); i++) {

  cv::Mat input_img = cv::imread(src_name[i], cv::IMREAD_COLOR);
  cout << "input_img width:" << input_img.cols << endl;

  torch::autograd::AutoGradMode guard(false);
  c10::IValue inputs = get_scripting_inputs(input_img, device);
  cout << "start infer" << endl;

  int pos = src_name[i].find_last_of(".");
  string new_name = src_name[i].substr(0, pos) + "_result.jpg";

  double time0 = static_cast<double>(getTickCount());
  auto outputs = module.forward({ inputs });
  time0 = ((double)getTickCount() - time0) / getTickFrequency();
  cout << "use time is " << time0 << endl;

} return 0; }

Expected behavior:

The code for single image inference runs normally，but Error encountered while using code for batch inference

the error is appeared when executing the forward function " auto outputs = module.forward({ inputs });"： 0x00007FFEE6F1CF19 处(位于 torchscript_mask_rcnn.exe 中)有未经处理的异常: Microsoft C++ 异常: c10::Error，位于内存位置 0x000000B8A5CFE2B0 处。

Environment:

win10+libtorch2.1.1+cuda12.1+pytorch2.1.1+opencv4.8

facebookresearch / detectron2