I want to use OpenCV to display RGB images, Depth images and Ir images captured by the camera. However, the expected effect cannot be achieved, and all created images cannot be displayed normally.

[LinkJam0]

I want to use OpenCV to display RGB images, Depth images and Ir images captured by the camera. However, the expected effect cannot be achieved, and all created images cannot be displayed normally.

This is my code

#pragma comment(lib, "k4a.lib")

#include <iostream>

#include <k4a/k4a.h>

#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core.hpp>
#include <opencv2/calib3d.hpp>

int main(int argc, char* argv, char* env) {

    uint32_t device_count = k4a_device_get_installed_count();
    printf("Found %d connect devices:\n", device_count);

    k4a_device_t device = NULL;
    if (K4A_RESULT_SUCCEEDED != k4a_device_open(K4A_DEVICE_DEFAULT, &device)) {
        printf("Failed to open device!\n");
        exit(1);
    }
    char* serial_number = NULL;
    size_t serial_number_length = 0;
    if (K4A_BUFFER_RESULT_TOO_SMALL != k4a_device_get_serialnum(device, NULL, &serial_number_length)) {
        printf("%lld: Failed to get serial number length!\n", serial_number_length);
        k4a_device_close(device);
        exit(1);
    }
    if (serial_number_length == NULL) {
        printf("Failed to allocate memory for serial number (%zu bytes)!\n", serial_number_length);
        k4a_device_close(device);
        device = NULL;
        exit(1);
    }
    serial_number = (char*)malloc(serial_number_length * sizeof(char));
    if (K4A_BUFFER_RESULT_SUCCEEDED != k4a_device_get_serialnum(device, serial_number, &serial_number_length)) {
        printf("Failed to get serial number!\n");
        free(serial_number);
        serial_number = NULL;
        k4a_device_close(device);
        device = NULL;
        exit(1);
    }
    else {
        printf("Opened device: %s\n", serial_number);
        free(serial_number);
    }

    k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL;
    config.camera_fps = K4A_FRAMES_PER_SECOND_30;
    config.color_format = K4A_IMAGE_FORMAT_COLOR_BGRA32;
    config.color_resolution = K4A_COLOR_RESOLUTION_720P;
    config.depth_mode = K4A_DEPTH_MODE_NFOV_UNBINNED;
    config.synchronized_images_only = true;

    if (K4A_RESULT_SUCCEEDED != k4a_device_start_cameras(device, &config)) {
        printf("Failed to start cameras!\n");
        k4a_device_close(device);
        exit(1);
    }
    if (K4A_RESULT_SUCCEEDED != k4a_device_start_imu(device)) {
        printf("Failed to start imu!\n");
        k4a_device_close(device);
        exit(1);
    }

    k4a_calibration_t sensor_calibration;
    if (K4A_RESULT_SUCCEEDED != k4a_device_get_calibration(device, config.depth_mode, config.color_resolution, &sensor_calibration)) {
        printf("Camera calibration error!");
        exit(1);
    }
    else {
        printf("Color camera calibration: resolution: %d x %d k4a_color_resolution_t: %d\n",
            sensor_calibration.color_camera_calibration.resolution_width,
            sensor_calibration.color_camera_calibration.resolution_height,
            sensor_calibration.color_resolution);
        printf("depth camera calibration: resolution: %d x %d depth_mode: %d\n",
            sensor_calibration.depth_camera_calibration.resolution_width,
            sensor_calibration.depth_camera_calibration.resolution_height,
            sensor_calibration.depth_mode);
    }

    k4a_capture_t sensor_capture = NULL;

    k4a_image_t sensor_depth_image = NULL;
    k4a_image_t sensor_color_image = NULL;
    k4a_image_t sensor_ir_image = NULL;

    uint8_t* depth_buf = nullptr;
    uint8_t* color_buf = nullptr;
    uint8_t* ir_buf = nullptr;

    k4a_imu_sample_t imu_sample;

    float temperature = NULL;

    cv::Mat color_image;
    cv::Mat depth_image;
    cv::Mat ir_image;

    for (;;) {
        switch (k4a_device_get_capture(device, &sensor_capture, K4A_WAIT_INFINITE)) {
        case K4A_WAIT_RESULT_SUCCEEDED: 
            break;
        case K4A_WAIT_RESULT_TIMEOUT:
            printf("Timed out waiting for a capture!\n");
            continue;
            break;
        case K4A_WAIT_RESULT_FAILED: 
            printf("Failed to read a capture!\n");
            k4a_capture_release(sensor_capture);
            goto EXIT;
        }

        switch (k4a_device_get_imu_sample(device, &imu_sample, K4A_WAIT_INFINITE)) {
        case K4A_WAIT_RESULT_SUCCEEDED: 
            break;
        case K4A_WAIT_RESULT_TIMEOUT:
            printf("Timed out waiting for a imu sample\n");
            continue;
            break;
        case K4A_WAIT_RESULT_FAILED:
            printf("Failed to read a imu sample\n");
            goto EXIT;
        }

        sensor_depth_image = k4a_capture_get_depth_image(sensor_capture);
        sensor_color_image = k4a_capture_get_color_image(sensor_capture);
        sensor_ir_image = k4a_capture_get_ir_image(sensor_capture);
        temperature = k4a_capture_get_temperature_c(sensor_capture);

        if (K4A_RESULT_SUCCEEDED != k4a_image_create(K4A_IMAGE_FORMAT_DEPTH16,
                                                k4a_image_get_width_pixels(sensor_depth_image),
                                                k4a_image_get_height_pixels(sensor_depth_image),
                                                k4a_image_get_stride_bytes(sensor_depth_image),
                                                &sensor_depth_image)) {
            printf("Failed to create depth image!\n");
            k4a_capture_release(sensor_capture);
            goto EXIT;
        }
        else {
            if (sensor_depth_image != NULL) {
                printf(" | Depth res:%4dx%4d stride:%5d\n",
                        k4a_image_get_width_pixels(sensor_depth_image),
                        k4a_image_get_height_pixels(sensor_depth_image),
                        k4a_image_get_stride_bytes(sensor_depth_image));
            }
        }
        if (K4A_RESULT_SUCCEEDED != k4a_image_create(K4A_IMAGE_FORMAT_COLOR_BGRA32,
                                                k4a_image_get_width_pixels(sensor_color_image),
                                                k4a_image_get_height_pixels(sensor_color_image),
                                                k4a_image_get_stride_bytes(sensor_color_image),
                                                &sensor_color_image)) {
            printf("Failed to create color image!\n\f");
            k4a_capture_release(sensor_capture);
            goto EXIT;
        }
        else {
            if (sensor_color_image != NULL) {
                printf(" | Color res:%4dx%4d stride:%5d\n",
                        k4a_image_get_width_pixels(sensor_color_image),
                        k4a_image_get_height_pixels(sensor_color_image),
                        k4a_image_get_stride_bytes(sensor_color_image));
            }
        }
        if (K4A_RESULT_SUCCEEDED != k4a_image_create(K4A_IMAGE_FORMAT_IR16,
                                                    k4a_image_get_width_pixels(sensor_ir_image),
                                                k4a_image_get_height_pixels(sensor_ir_image),
                                                k4a_image_get_stride_bytes(sensor_ir_image),
                                                &sensor_ir_image)) {
            printf("Failed to create color image!\n\f");
            k4a_capture_release(sensor_capture);
            goto EXIT;
        }
        else {
            if (sensor_ir_image != NULL) {
                printf(" | Ir res:%4dx%4d stride:%5d\n",
                        k4a_image_get_width_pixels(sensor_ir_image),
                        k4a_image_get_height_pixels(sensor_ir_image),
                        k4a_image_get_stride_bytes(sensor_ir_image));
            }
        }

        printf(" | Temperature: %.2f\n | accelerometer x:%.4f yc:%.4f z: %.4f\n | gyroscope x:%.4f y:%.4f z: %.4f\n",
                imu_sample.temperature,
                imu_sample.acc_sample.xyz.x,
                imu_sample.acc_sample.xyz.y,
                imu_sample.acc_sample.xyz.z,
                imu_sample.gyro_sample.xyz.x,
                imu_sample.gyro_sample.xyz.y,
                imu_sample.gyro_sample.xyz.z);

        printf(" | Device temperature: %.2f\n", temperature);

        printf("depth format: %d\n", k4a_image_get_format(sensor_depth_image));
        printf("color format: %d\n", k4a_image_get_format(sensor_color_image));
        printf("ir format: %d\n", k4a_image_get_format(sensor_ir_image));
        printf("depth size: %I64d\n", k4a_image_get_size(sensor_depth_image));
        printf("color size: %I64d\n", k4a_image_get_size(sensor_color_image));
        printf("ir size: %I64d\n", k4a_image_get_size(sensor_ir_image));
        printf("color iso: %I32d\n", k4a_image_get_iso_speed(sensor_color_image));
        printf("white balance: %I32d\n", k4a_image_get_white_balance(sensor_color_image));
        printf("exposure usec: %I64d\n", k4a_image_get_exposure_usec(sensor_color_image));

        depth_buf = k4a_image_get_buffer(sensor_depth_image);
        color_buf = k4a_image_get_buffer(sensor_color_image);
        ir_buf = k4a_image_get_buffer(sensor_ir_image);

        depth_image = cv::Mat(k4a_image_get_height_pixels(sensor_depth_image),
            k4a_image_get_width_pixels(sensor_depth_image),
            CV_8UC1,
            (void*)depth_buf,
            cv::Mat::AUTO_STEP);
        color_image = cv::Mat(k4a_image_get_height_pixels(sensor_color_image),
            k4a_image_get_width_pixels(sensor_color_image),
            CV_8UC4,
            (void*)color_buf,
            cv::Mat::AUTO_STEP);
        ir_image = cv::Mat(k4a_image_get_height_pixels(sensor_ir_image),
            k4a_image_get_width_pixels(sensor_ir_image),
            CV_8UC1,
            (void*)ir_buf,
            cv::Mat::AUTO_STEP);

        cv::imshow("depth", depth_image);
        cv::imshow("color", color_image);
        cv::imshow("ir", ir_image);
        cv::waitKey();

        cv::imwrite("depth.jpg", depth_image);
        cv::imwrite("color.jpg", color_image);
        cv::imwrite("ir.jpg", ir_image);

        depth_image.release();
        color_image.release();
        ir_image.release();

        k4a_image_release(sensor_depth_image);
        k4a_image_release(sensor_color_image);
        k4a_image_release(sensor_ir_image);

    EXIT:
        k4a_capture_release(sensor_capture);
    }

    k4a_device_stop_cameras(device);
    k4a_device_stop_imu(device);
    k4a_device_close(device);

    return 0;
}

The operation results are shown in the figure below

Desktop:

• Azure Kinect SDK Version: v1.4.1
• OpenCV Version: 4.6.0
• Visual Studio 2022: v143

I tried many solutions and even replaced OpenCV3, but still could not read the image information of the camera. I noticed that when my code was running, the exposure usec, iso and white balance value in the parameters of the color camera was all zero. Is this caused by the camera not being started correctly? But k4a device start Cameras function() and k4a device get The calibration() function returned correct results. I don't know how to solve it. I hope i can get your help. thanks.

[dasparli]

You are setting the variable sensor_depth_image with a call to k4a_capture_get_depth_image() and then overwriting that same variable with a subsequent call to k4a_image_create().

[Risekj]

Try my code, I was in the same situation as you before, and then after the modification, it can display RGB, Depth, and Ir window images.But I didn't optimize the image further in this code, you can see if it can be displayed properly. SDK1.4.1 VS2022 OPENCV4.5.5

`#include <k4a/k4a.h>

include <opencv2/opencv.hpp>

int main() { k4a_device_t device = nullptr;

// 打开 Azure Kinect 设备
// (k4a_device_start_cameras(device, &config): 启动相机，
通过传递设备和配置信息来启用相机的彩色、深度和红外流。这一步包含启动所有已配置的流，
包括 RGB、深度和红外。
if (K4A_FAILED(k4a_device_open(K4A_DEVICE_DEFAULT, &device)))
{
    printf("无法打开 Azure Kinect 设备！\n");
    return 1;
}

// 配置设备启用深度和红外流
k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL;
config.camera_fps = K4A_FRAMES_PER_SECOND_15;
config.depth_mode = K4A_DEPTH_MODE_WFOV_2X2BINNED;  // 设置深度模式
config.color_format = K4A_IMAGE_FORMAT_COLOR_BGRA32;
config.color_resolution = K4A_COLOR_RESOLUTION_1080P;

// 启动相机
if (K4A_FAILED(k4a_device_start_cameras(device, &config)))
{
    printf("无法启动相机！\n");
    k4a_device_close(device);
    return 1;
}

// 创建 OpenCV 窗口
cv::namedWindow("Color Image", cv::WINDOW_NORMAL);
cv::namedWindow("Depth Image", cv::WINDOW_NORMAL);
cv::namedWindow("IR Image", cv::WINDOW_NORMAL);

while (true)
{
    k4a_capture_t capture = nullptr;

    // 捕获一帧
    if (K4A_SUCCEEDED(k4a_device_get_capture(device, &capture, K4A_WAIT_INFINITE)))
    {
        // 获取彩色图像
        k4a_image_t colorImage = k4a_capture_get_color_image(capture);
        if (colorImage != nullptr)
        {
            // 处理并显示彩色图像（使用 OpenCV）
            cv::Mat colorMat(k4a_image_get_height_pixels(colorImage), k4a_image_get_width_pixels(colorImage),
                             CV_8UC4, (void*)k4a_image_get_buffer(colorImage));
            cv::imshow("Color Image", colorMat);

            k4a_image_release(colorImage);
        }

        // 获取深度图
        k4a_image_t depth_image = k4a_capture_get_depth_image(capture);
        if (depth_image != nullptr)
        {
            // 处理并显示深度图（使用 OpenCV）
            int depth_width = k4a_image_get_width_pixels(depth_image);
            int depth_height = k4a_image_get_height_pixels(depth_image);
            cv::Mat depth_mat(depth_height, depth_width, CV_16U, (void*)k4a_image_get_buffer(depth_image));
            cv::Mat depth_display;
            cv::normalize(depth_mat, depth_display, 0, 255, cv::NORM_MINMAX, CV_8U);
            cv::imshow("Depth Image", depth_display);

            k4a_image_release(depth_image);
        }

        // 获取红外图像
        k4a_image_t ir_image = k4a_capture_get_ir_image(capture);
        if (ir_image != nullptr)
        {
            // 处理并显示红外图像（使用 OpenCV）
            cv::Mat ir_mat(k4a_image_get_height_pixels(ir_image), k4a_image_get_width_pixels(ir_image),
                           CV_16U, (void*)k4a_image_get_buffer(ir_image));
            cv::Mat ir_8bit;
            ir_mat.convertTo(ir_8bit, CV_8U);
            cv::imshow("IR Image", ir_8bit);

            k4a_image_release(ir_image);
        }

        // 释放捕获
        k4a_capture_release(capture);

        // 检查退出键盘按键
        if (cv::waitKey(1) == 27) // ESC键
            break;
    }
}

// 停止相机并关闭设备
k4a_device_stop_cameras(device);
k4a_device_close(device);

// 关闭 OpenCV 窗口
cv::destroyAllWindows();

return 0;

} `