请问如何将绿色背景替换为图片呢？

[cvDFT]

需要对图片进行怎样的数据处理

[DefTruth]

可以参考我在lite.ai.toolkit的一个回答：

• https://github.com/DefTruth/lite.ai.toolkit/issues/223

cv::Mat out_mat = fgr_mat.mul(three_channel_pha_mat) + bgr_mat.mul(1. - three_channel_pha_mat);

应该是用类似的逻辑，opencv的mul方法支持相同size大小的Mat进行逐元素运算。或者先将fgr_mat和img_bgr用split成三个通道，逐个通道运算后合并就行。

std::vector<cv::Mat> fgr_mat_channels, bgr_mat_channels;
cv::split(fgr_mat, fgr_mat_channels);
cv::split(img_bgr, bgr_mat_channels);
auto rmat = fgr_mat_channels.at(0);
auto bmat = fgr_mat_channels.at(1);
auto gmat = fgr_mat_channels.at(2);
auto brmat = bgr_mat_channels.at(0);
auto bbmat = bgr_mat_channels.at(1);
auto bgmat = bgr_mat_channels.at(2);
cv::Mat rest = 1. - pha_mat;
cv::Mat mbmat = bmat.mul(pmat) + rest.mul(brmat);
cv::Mat mgmat = gmat.mul(pmat) + rest.mul(bbmat);
cv::Mat mrmat = rmat.mul(pmat) + rest.mul(bgmat);
std::vector<cv::Mat> merge_channel_mats;
merge_channel_mats.push_back(mbmat);
merge_channel_mats.push_back(mgmat);
merge_channel_mats.push_back(mrmat);
cv::Mat merge_mat;
cv::merge(merge_channel_mats, merge_mat);

这是处理方式的问题。另外，后处理可以考虑增加保留最大连通区域的算法，去除一些小的黑点。这个后处理算法可以参考我的文章，就不这里解释原理了。

• 知乎文章：https://zhuanlan.zhihu.com/p/442949027

  
  // https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L208
  void MGMatting::remove_small_connected_area(cv::Mat &alpha_pred)
  {
  cv::Mat gray, binary;
  alpha_pred.convertTo(gray, CV_8UC1, 255.f);
  // 255 * 0.05 ~ 13
  // https://github.com/yucornetto/MGMatting/blob/main/code-base/utils/util.py#L209
  cv::threshold(gray, binary, 13, 255, cv::THRESH_BINARY);
  // morphologyEx with OPEN operation to remove noise first.
  auto kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(3, 3), cv::Point(-1, -1));
  cv::morphologyEx(binary, binary, cv::MORPH_OPEN, kernel);
  // Computationally connected domain
  cv::Mat labels = cv::Mat::zeros(alpha_pred.size(), CV_32S);
  cv::Mat stats, centroids;
  int num_labels = cv::connectedComponentsWithStats(binary, labels, stats, centroids, 8, 4);
  if (num_labels <= 1) return; // no noise, skip.
  // find max connected area, 0 is background
  int max_connected_id = 1; // 1,2,...
  int max_connected_area = stats.at<int>(max_connected_id, cv::CC_STAT_AREA);
  for (int i = 1; i < num_labels; ++i)
  {
  int tmp_connected_area = stats.at<int>(i, cv::CC_STAT_AREA);
  if (tmp_connected_area > max_connected_area)
  {
    max_connected_area = tmp_connected_area;
    max_connected_id = i;
  }
  }
  const int h = alpha_pred.rows;
  const int w = alpha_pred.cols;
  // remove small connected area.
  for (int i = 0; i < h; ++i)
  {
  int *label_row_ptr = labels.ptr<int>(i);
  float *alpha_row_ptr = alpha_pred.ptr<float>(i);
  for (int j = 0; j < w; ++j)
  {
    if (label_row_ptr[j] != max_connected_id)
      alpha_row_ptr[j] = 0.f;
  }
  }
  }

这个alpha_pred就是预测的pha_mat;

[hanxizai]

请问您修改成功了吗

[DefTruth]

可以参考以下这段逻辑，我在lite里面增加了一些辅助函数，但还没合并进主分支，你可以参考下：

void lite::utils::swap_background(const cv::Mat &fgr_mat, const cv::Mat &pha_mat,
                                  const cv::Mat &bgr_mat, cv::Mat &out_mat,
                                  bool fgr_is_already_mul_pha)
{
  // user-friendly method for background swap.
  if (fgr_mat.empty() || pha_mat.empty() || bgr_mat.empty()) return;
  const unsigned int fg_h = fgr_mat.rows;
  const unsigned int fg_w = fgr_mat.cols;
  const unsigned int bg_h = bgr_mat.rows;
  const unsigned int bg_w = bgr_mat.cols;
  const unsigned int ph_h = pha_mat.rows;
  const unsigned int ph_w = pha_mat.cols;
  const unsigned int channels = fgr_mat.channels();
  if (channels != 3) return; // only support 3 channels.
  const unsigned int num_elements = fg_h * fg_w * channels;

  cv::Mat bg_mat_copy, ph_mat_copy, fg_mat_copy;
  if (bg_h != fg_h || bg_w != fg_w)
    cv::resize(bgr_mat, bg_mat_copy, cv::Size(fg_w, fg_h));
  else bg_mat_copy = bgr_mat; // ref only.
  if (ph_h != fg_h || ph_w != fg_w)
    cv::resize(pha_mat, ph_mat_copy, cv::Size(fg_w, fg_h));
  else ph_mat_copy = pha_mat; // ref only.
  if (ph_mat_copy.channels() == 1)
    cv::cvtColor(ph_mat_copy, ph_mat_copy, cv::COLOR_GRAY2BGR); // 0.~1.
  // convert mats to float32 points.
  if (bg_mat_copy.type() != CV_32FC3) bg_mat_copy.convertTo(bg_mat_copy, CV_32FC3); // 0.~255.
  if (ph_mat_copy.type() != CV_32FC3) ph_mat_copy.convertTo(ph_mat_copy, CV_32FC3); // 0.~1.
  if (fgr_mat.type() != CV_32FC3) fgr_mat.convertTo(fg_mat_copy, CV_32FC3); // 0.~255.
  else fg_mat_copy = fgr_mat; // ref only

  // element wise operations.
  out_mat = fg_mat_copy.clone();
  const float *fg_ptr = (float *) fg_mat_copy.data;
  const float *bg_ptr = (float *) bg_mat_copy.data;
  const float *ph_ptr = (float *) ph_mat_copy.data;
  float *mutable_out_ptr = (float *) out_mat.data;

  // TODO: add omp support instead of native loop.
  if (!fgr_is_already_mul_pha)
    for (unsigned int i = 0; i < num_elements; ++i)
      mutable_out_ptr[i] = fg_ptr[i] * ph_ptr[i] + (1.f - ph_ptr[i]) * bg_ptr[i];
  else
    for (unsigned int i = 0; i < num_elements; ++i)
      mutable_out_ptr[i] = fg_ptr[i] + (1.f - ph_ptr[i]) * bg_ptr[i];

  if (!out_mat.empty() && out_mat.type() != CV_8UC3)
    out_mat.convertTo(out_mat, CV_8UC3);
}

使用案例（MODNet还在开发中，此处仅用作参考示例）

static void test_default()
{
  std::string onnx_path = "../../../hub/onnx/cv/modnet_photographic_portrait_matting-512x512.onnx";
  std::string test_img_path = "../../../examples/lite/resources/test_lite_matting_input.jpg";
  std::string test_bgr_path = "../../../examples/lite/resources/test_lite_matting_bgr.jpg";
  std::string save_fgr_path = "../../../logs/test_lite_modnet_fgr.jpg";
  std::string save_pha_path = "../../../logs/test_lite_modnet_pha.jpg";
  std::string save_merge_path = "../../../logs/test_lite_modnet_merge.jpg";
  std::string save_swap_path = "../../../logs/test_lite_modnet_swap.jpg";

  lite::cv::matting::MODNet *modnet =
      new lite::cv::matting::MODNet(onnx_path, 16); // 16 threads

  lite::types::MattingContent content;
  cv::Mat img_bgr = cv::imread(test_img_path);
  cv::Mat bgr_mat = cv::imread(test_bgr_path);

  // 1. image matting.
  modnet->detect(img_bgr, content, true);

  if (content.flag)
  {
    if (!content.fgr_mat.empty()) cv::imwrite(save_fgr_path, content.fgr_mat);
    if (!content.pha_mat.empty()) cv::imwrite(save_pha_path, content.pha_mat * 255.);
    if (!content.merge_mat.empty()) cv::imwrite(save_merge_path, content.merge_mat);
    // swap background
    cv::Mat out_mat;
    lite::utils::swap_background(content.fgr_mat, content.pha_mat, bgr_mat, out_mat, true);
    if (!out_mat.empty())
    {
      cv::imwrite(save_swap_path, out_mat);
      std::cout << "Saved Swap Image Done!" << std::endl;
    }

    std::cout << "Default Version MGMatting Done!" << std::endl;
  }

  delete modnet;
}

效果示例

• 合成图
• 原图
• 背景图