Struggling with descriptor matching

[dominikheinz]

System Info:

• Operating system: Arch Linux
• The way you installed OpenCV: pacman
• OpenCV version: opencv 4.9.0-2
• rustc version (rustc --version): rustc 1.74.0 (79e9716c9 2023-11-13)

Problem Description: I am attempting to replicate these two OpenCV examples with opencv-rust.

• https://docs.opencv.org/3.4/dc/dc3/tutorial_py_matcher.html
• https://docs.opencv.org/3.4/d5/d6f/tutorial_feature_flann_matcher.html

I have two images (1.jpeg and 2.jpeg) with visual overlaps. While successfully extracting keypoints from both images, I fail to find matches between the keypoints using the knn_match_def function.

This is my code so far:

use anyhow::Context;
use cv::{
    core::{DMatch, InputArray, Vector, NORM_L2},
    features2d::{DescriptorMatcher, DescriptorMatcher_FLANNBASED},
    types,
};
use opencv::{self as cv, prelude::*};

fn main() {
    // Read images
    let img = match cv::imgcodecs::imread(
        "/data/1.jpeg",
        cv::imgcodecs::IMREAD_COLOR,
    ) {
        Ok(img) => img,
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    };

    let img2 = match cv::imgcodecs::imread(
        "/data/2.jpeg",
        cv::imgcodecs::IMREAD_COLOR,
    ) {
        Ok(img2) => img2,
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    };

    // Setup SIFT
    let mask = cv::core::Mat::default();
    let mask2 = cv::core::Mat::default();
    let mut sift = match cv::features2d::SIFT::create(0, 3, 0.04, 10., 1.6, false) {
        Ok(sift) => sift,
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    };
    let mut keypoints1 = cv::core::Vector::default();
    let mut descriptors1 = cv::core::Mat::default();

    let mut keypoints2 = cv::core::Vector::default();
    let mut descriptors2 = cv::core::Mat::default();

    // Save original images
    match cv::imgcodecs::imwrite("./output.jpeg", &img, &cv::core::Vector::default()) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }
    match cv::imgcodecs::imwrite("./output2.jpeg", &img, &cv::core::Vector::default()) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }

    // Compute keypoints using SIFT
    match sift.detect_and_compute(&img, &mask, &mut keypoints1, &mut descriptors1, false) {
        Ok(_) => {
            // println!("Keypoints: {:#?}", keypoints1);
            println!("Keypoints1 Count: {}", keypoints1.len());
        }
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }
    match sift.detect_and_compute(&img2, &mask2, &mut keypoints2, &mut descriptors2, false) {
        Ok(_) => {
            // println!("Keypoints: {:#?}", keypoints2);
            println!("Keypoints2 Count: {}", keypoints2.len());
        }
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }

    // Draw keypoints into the image
    println!("Visualizing keypoints ...");
    let mut dst_img = cv::core::Mat::default();
    let mut dst_img2 = cv::core::Mat::default();
    match cv::features2d::draw_keypoints(
        &img,
        &keypoints1,
        &mut dst_img,
        cv::core::VecN([0., 255., 0., 255.]),
        cv::features2d::DrawMatchesFlags::DEFAULT,
    ) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }
    match cv::features2d::draw_keypoints(
        &img2,
        &keypoints2,
        &mut dst_img2,
        cv::core::VecN([0., 255., 0., 255.]),
        cv::features2d::DrawMatchesFlags::DEFAULT,
    ) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }

    // Export images that have keypoints drawn into them
    match cv::imgcodecs::imwrite("./1-keypoints.jpeg", &dst_img, &cv::core::Vector::default()) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }
    match cv::imgcodecs::imwrite("./2-keypoints.jpeg", &dst_img2, &cv::core::Vector::default()) {
        Ok(_) => (),
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    }

    // Match keypoints together
    let mut descs: Vector<Mat> = Vector::default();
    descs.push(descriptors1);
    descs.push(descriptors2);
    let mut knn_matches: Vector<Vector<DMatch>> = Vector::default();
    let mut matcher = match DescriptorMatcher::create("FlannBased") {
        Ok(m) => m,
        Err(e) => {
            println!("Error: {}", e);
            return;
        }
    };

    // Why will it not find any matches, shouldn't it populate?
    let res = matcher.knn_match_def(&descs, &mut knn_matches, 2);
    println!("Matches: {}", knn_matches.len());
    println!("Successfully ran knn matching");
    println!("knn_matches: {:#?}", knn_matches);
    println!("Finished SIFT keypoint identification.");
}

The matches vector is empty. (I assume it would be populated by the knn_match_def/knn_match fn?)

fn knn_match(
    &mut self,
    query_descriptors: &impl ToInputArray,
    matches: &mut Vector<Vector<DMatch>>,
    k: i32,
    masks: &impl ToInputArray,
    compact_result: bool
) -> Result<()>

I am unsure about how to correctly pass the descriptors as an InputArray. In the C++ example, they pass descriptors as individual parameters. Probably I am just not using the library correct, and I am not passing the descriptors correctly. However, I don't know how to do it correctly? I assume ideally I should use the knn_match function directly? That being said, I still don't know how I would pass these parameters correctly (_querydescriptors, matches and masks).

Hopefully someone can give me some pointers on what I am doing wrong, and how I can fix my code. Thanks :)

[markhilb]

You need to use the knn_train_match/knn_train_match_def function.

matcher.knn_train_match_def(&descriptors1, &descriptors1, &mut knn_matches, 2);