SoloV2 C++ deploy

[lfs119]

Is there .cc file like object_detection.cc for deploying SoloV2? I just find Python version supports model arch of SoloV2. Looking forward to your reply.

[yghstill]

@lfs119 Currently only supports python deploying inference, C++ side does not yet support.

[lfs119]

Thanks for your response and Is there a plan for supporting c++ deploying inference?

[gsdust]

// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License.

include

include

include

include

include

include

include

include "paddle_inference_api.h"

include "yaml-cpp/yaml.h"

include <opencv2/core/core.hpp>

include <opencv2/imgproc/imgproc.hpp>

include <opencv2/highgui/highgui.hpp>

class ImageBlob { public: // Original image width and height std::vector ori_imsize; // Buffer for image data after preprocessing std::vector imdata; // Original image width, height, shrink in float format std::vector ori_im_sizef; // Evaluation image width and height std::vector eval_im_sizef; // Scale factor for image size to origin image size std::vector scale_factorf; };

struct ObjectResult { // Rectangle coordinates of detected object: left, right, top, down std::vector rect; // Class id of detected object int class_id; // Confidence of detected object float confidence; };

struct SOLOResult { // Rectangle coordinates of detected object: left, right, top, down std::vector point; // Class id of detected object int class_id; // Confidence of detected object float confidence; std::vector maskImg; };

std::vector outputdata;

using namespace paddle_infer;

void CreateConfig(Config config, const std::string& model_dirname) { // 模型从磁盘进行加载 config->SetModel(model_dirname + "model", model_dirname + "params"); // config->SetModel(model_dirname); // 如果模型从内存中加载，可以使用SetModelBuffer接口 // config->SetModelBuffer(prog_buffer, prog_size, params_buffer, params_size); config->EnableUseGpu(1000 /设定GPU初始显存池为MB/, 0 /设定GPU ID为0*/); //开启GPU预测

/* for cpu
config->DisableGpu();
config->EnableMKLDNN();   // 开启MKLDNN加速
config->SetCpuMathLibraryNumThreads(10);
*/

config->SwitchIrDebug(true);         // 可视化调试选项，若开启，则会在每个图优化过程后生成dot文件
config->SwitchIrOptim(false);     // 默认为true。如果设置为false，关闭所有优化
// config->EnableMemoryOptim();     // 开启内存/显存复用

}

void Normalize(cv::Mat& im) { double e = 1.0; e /= 255.0; (im).convertTo(im, CV_32FC3, e); for (int h = 0; h < im.rows; h++) { for (int w = 0; w < im.cols; w++) { im.at(h, w)[0] = (im.at(h, w)[0] - 0.485) / 0.229; im.at(h, w)[1] = (im.at(h, w)[1] - 0.456) / 0.224; im.at(h, w)[2] = (im.at(h, w)[2] - 0.406) / 0.225; } } }

cv::Mat VisualizeResult(const cv::Mat& img, const std::vector& results, const std::vector& colormap) { cv::Mat vis_img = img.clone(); for (int i = 0; i < results.size(); ++i) { int w = results[i].rect[1] - results[i].rect[0]; int h = results[i].rect[3] - results[i].rect[2]; cv::Rect roi = cv::Rect(results[i].rect[0], results[i].rect[2], w, h);

    // Configure color and text size
    std::ostringstream oss;
    oss << std::setiosflags(std::ios::fixed) << std::setprecision(4);
    oss << i << " ";
    oss << results[i].confidence;
    std::string text = oss.str();
    int c1 = colormap[3 * results[i].class_id + 0];
    int c2 = colormap[3 * results[i].class_id + 1];
    int c3 = colormap[3 * results[i].class_id + 2];
    cv::Scalar roi_color = cv::Scalar(c1, c2, c3);
    int font_face = cv::FONT_HERSHEY_COMPLEX_SMALL;
    double font_scale = 0.5f;
    float thickness = 0.5;
    cv::Size text_size = cv::getTextSize(text,
        font_face,
        font_scale,
        thickness,
        nullptr);
    cv::Point origin;
    origin.x = roi.x;
    origin.y = roi.y;

    // Configure text background
    cv::Rect text_back = cv::Rect(results[i].rect[0],
        results[i].rect[2] - text_size.height,
        text_size.width,
        text_size.height);

    // Draw roi object, text, and background
    cv::rectangle(vis_img, roi, roi_color, 2);
    cv::rectangle(vis_img, text_back, roi_color, -1);
    cv::putText(vis_img,
        text,
        origin,
        font_face,
        font_scale,
        cv::Scalar(255, 255, 255),
        thickness);
}
return vis_img;

}

void Permute(cv::Mat& im, ImageBlob data) { int rh = im.rows; int rw = im.cols; int rc = im.channels(); (data->imdata).resize(rc rh rw); float base = (data->imdata).data(); for (int i = 0; i < rc; ++i) { cv::extractChannel(im, cv::Mat(rh, rw, CV_32FC1, base + i rh rw), i); } }

void Postprocess(std::vector outputdata,std::vector &result) { result.clear(); int rh = 1; int rw = 1;

int total_size = output_data_.size() / 6;
for (int j = 0; j < total_size; ++j) {
    // Class id
    int class_id = static_cast<int>(round(output_data_[0 + j * 6]));
    // Confidence score
    float score = output_data_[1 + j * 6];
    int xmin = (output_data_[2 + j * 6] * rw);
    int ymin = (output_data_[3 + j * 6] * rh);
    int xmax = (output_data_[4 + j * 6] * rw);
    int ymax = (output_data_[5 + j * 6] * rh);
    int wd = xmax - xmin;
    int hd = ymax - ymin;
    if (score > 0.5 && class_id > -1) {
        ObjectResult result_item;
        result_item.rect = { xmin, xmax, ymin, ymax };
        result_item.class_id = class_id;
        result_item.confidence = score;
        result.push_back(result_item);
    }
}

}

void Postprocess(std::vector confidence, std::vector label, std::vector< int> masks, std::vector& result) { result.clear(); int total_size = label.size() - 1;

for (size_t j = 0; j < total_size; ++j) {
    // Class id
    //int class_id = static_cast<int>(round(output_data_[0 + j * 6]));
    int class_id = label[j];

    // Confidence score
    float score = confidence[j];
    if (score > 0.5 && class_id > -1) {
        SOLOResult result_item;
        cv::Mat resultMat=cv::Mat::zeros(512, 512, CV_8UC1);
        for (size_t i = 0; i < 512; i++)
        {
            for (size_t k = 0; k < 512; k++)
            {
                //std::cout<<
                if (masks[j*512*512+i*512+k]!= 0 )
                {
                    //std::cout << masks[j * 512 * 512 + i * 512 + k] << std::endl;
                    cv::Point point = cv::Point(i,k);
                    resultMat.at<uchar>(i, k) = (char)class_id;
                    result_item.point.push_back(point);
                }

            }

        }
        result_item.class_id = class_id;
        result_item.confidence = score;
        result.push_back(result_item);
    }
}

}

std::vector GenerateColorMap(int num_class) { auto colormap = std::vector(3 num_class, 0); for (int i = 0; i < num_class; ++i) { int j = 0; int lab = i; while (lab) { colormap[i 3] |= (((lab >> 0) & 1) << (7 - j)); colormap[i 3 + 1] |= (((lab >> 1) & 1) << (7 - j)); colormap[i 3 + 2] |= (((lab >> 2) & 1) << (7 - j)); ++j; lab >>= 3; } } return colormap; }

void RunAnalysis() { // 1. create AnalysisConfig

std::string model_dirname = "/";
//std::string model_dirname = "D:\\PaddleDetection\\inference_model\\solov2_light_r50_vd_fpn_dcn_512_3x\\";
Config config;
CreateConfig(&config, model_dirname);
// CreateConfig(&config);

config.EnableUseGpu(4096, 0);
std::cout << "GPUID" << config.gpu_device_id() << std::endl;;
// use ZeroCopyTensor, Must be set to false
config.SwitchUseFeedFetchOps(false);

ImageBlob inputB;
// 2. create predictor, prepare input data
auto predictor = CreatePredictor(config);
int batch_size = 1;
int channels = 3;
int height = 512;
int width = 512;
int nums = batch_size * channels * height * width;

std::string imageDir = "/";
cv::Mat src = cv::imread(imageDir, 1);
cv::Mat RGBImg,resizeImg;
cv::cvtColor(src, RGBImg, cv::COLOR_RGB2BGR);

cv::resize(RGBImg, resizeImg, cv::Size(512, 512));
Normalize(resizeImg);
Permute(resizeImg, &inputB);

inputB.eval_im_size_f_ = { 512,512,1.0 };
// 3. create input tensor, use ZeroCopyTensor

inputB.ori_im_size_ = { src.cols,src.rows };
auto input_names = predictor->GetInputNames();
for (const auto& tensor_name : input_names) {
    auto in_tensor = predictor->GetInputHandle(tensor_name);
    if (tensor_name == "image") {
        in_tensor->Reshape({ 1, 3, height, width });
        in_tensor->CopyFromCpu(inputB.im_data_.data());
    }
    else if (tensor_name == "im_size") {
        in_tensor->Reshape({ 1, 2 });
        in_tensor->CopyFromCpu(inputB.ori_im_size_.data());
    }
    else if (tensor_name == "im_info") {
        in_tensor->Reshape({ 1, 3 });
        in_tensor->CopyFromCpu(inputB.eval_im_size_f_.data());
    }
    else if (tensor_name == "im_shape") {
        in_tensor->Reshape({ 1, 3 });
        in_tensor->CopyFromCpu(inputB.ori_im_size_f_.data());
    }
    else if (tensor_name == "scale_factor") {
        in_tensor->Reshape({ 1, 4 });
        in_tensor->CopyFromCpu(inputB.scale_factor_f_.data());
    }
}

// 4. run predictor
//predictor->ZeroCopyRun();

predictor->Run();
// 5. get out put

std::vector<SOLOResult> resultSolo;
std::vector<float> out_data;
std::vector<int64_t> out_data0;
std::vector<int> out_data2;
auto output_names = predictor->GetOutputNames();
auto output_t = predictor->GetOutputHandle(output_names[1]);
auto output_t0 = predictor->GetOutputHandle(output_names[0]);
auto output_t2 = predictor->GetOutputHandle(output_names[2]);

std::vector<int> output_shape = output_t->shape();
int out_num = std::accumulate(output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
out_data.resize(out_num);
output_t->CopyToCpu(out_data.data());

std::vector<int> output_shape0 = output_t0->shape();
int out_num0 = std::accumulate(output_shape0.begin(), output_shape0.end(), 1, std::multiplies<int>());
out_data0.resize(out_num0);
output_t0->CopyToCpu(out_data0.data());

std::vector<int> output_shape2 = output_t2->shape();
int out_num2 = std::accumulate(output_shape2.begin(), output_shape2.end(), 1, std::multiplies<int>());
out_data2.resize(out_num2);
output_t2->CopyToCpu(out_data2.data());

Postprocess(out_data, out_data0, out_data2, resultSolo);

//yolo
//std::vector<ObjectResult> result;
//std::vector<ObjectResult> result;
//std::vector<float> out_data;
//std::vector<int64_t> out_data0;
//std::vector<int> out_data2;
//auto output_names = predictor->GetOutputNames();
//auto output_t = predictor->GetOutputHandle(output_names[0]);
//std::vector<int> output_shape = output_t->shape();
//int out_num = std::accumulate(output_shape.begin(), output_shape.end(), 1, std::multiplies<int>());
//out_data.resize(out_num);
//output_t->CopyToCpu(out_data.data());

//auto output_names = predictor->GetOutputNames();
//auto output_t = predictor->GetOutputHandle(output_names[0]);

//std::vector<int> output_shape = output_t->shape();
//int output_size = 1;
//for (int j = 0; j < output_shape.size(); ++j) {
//  output_size *= output_shape[j];
//}

//if (output_size < 6) {
//  std::cerr << "[WARNING] No object detected." << std::endl;
//}
//std::vector<ObjectResult> result;
//output_data_.resize(output_size);
//output_t->CopyFromCpu(output_data_.data());

//Postprocess(out_data,result);

//for (const auto& item : result) {
//  printf("class=%d confidence=%.4f rect=[%d %d %d %d]\n",
//      item.class_id,
//      item.confidence,
//      item.rect[0],
//      item.rect[1],
//      item.rect[2],
//      item.rect[3]);
//}

cv::Mat aaaaa;
//auto colormap = GenerateColorMap(81);
//std::string output_dir = "D:\\test.jpg";
//cv::Mat vis_img = VisualizeResult(
//  src, result,  colormap);
//cv::imwrite(output_dir, vis_img);

}

int main(int argc, char** argv) { RunAnalysis(); std::cout << "=========================Runs successfully====================" << std::endl; return 0; }