Question about depth

[DQSSSSS]

Hi,

Thanks for your great work! I tried to generate my scenes' dataset to test on your model. I got depth by multiplying matrix camera_matrices[i] (in $(dataset)/inputs/camera_matrices.h) left by each pixel of i-th world_position, then normalized z-coordinate, but the results are different to yours.

Could you tell me how to generate depth data, or give me your data maker code? Thank you!

[xmeng525]

Hi, I followed the same process. I've attached my code for generating the depth images. Please let me know if you can generate the depth image using the same process. `

define _CRT_SECURE_NO_WARNINGS

include

include "EXRImage.h"

include <glm/glm.hpp>

include "sponza\inputs\camera_matrices.h"

include <opencv2/core.hpp>

include <opencv2/imgcodecs.hpp>

include <opencv2/highgui.hpp>

include "opencv2/imgproc/imgproc.hpp"

int main() { std::string scene_name = "sponza"; std::string main_dir = "E:\FoveatedPathTracing\BorrowedProjects\Blockwise"; std::string data_dir = main_dir + "\" + "BMFR-dataset-fixed01-part1of2"; std::string scene_dir = data_dir + "\" + scene_name + "\" + "inputs" + "\";

std::string output_dir = main_dir + "\\" + "results" + "\\" + scene_name;
std::system(("mkdir " + output_dir).c_str());
std::string depth_subdir = output_dir + "\\" + "depth_normalized";
std::system(("mkdir " + depth_subdir).c_str());

std::ofstream myfile;
myfile.open(depth_subdir + "\\max_min_depth.txt");
myfile << "max & min depth in camera coordinates. \n";
int data_size = 60;
EXRImage *exr_helper = new EXRImage();
for (int index = 0; index < data_size; index++)
{
    EXRInfo image = exr_helper->LoadEXR(scene_dir + "world_position" + std::to_string(index) + ".exr");
    glm::mat4 cam = glm::mat4(
        camera_matrices[index][0][0],
        camera_matrices[index][0][1],
        camera_matrices[index][0][2],
        camera_matrices[index][0][3],
        camera_matrices[index][1][0],
        camera_matrices[index][1][1],
        camera_matrices[index][1][2],
        camera_matrices[index][1][3],
        camera_matrices[index][2][0],
        camera_matrices[index][2][1],
        camera_matrices[index][2][2],
        camera_matrices[index][2][3],
        camera_matrices[index][3][0],
        camera_matrices[index][3][1],
        camera_matrices[index][3][2],
        camera_matrices[index][3][3]
    );

    float *camera_coords = new float[4 * image.height * image.width];
    float max_depth = INT_MIN;
    float min_depth = INT_MAX;
    for (int y = 0; y < image.height; y++)
    {
        for (int x = 0; x < image.width; x++)
        {
            int pixel_index = y * image.width + x;
            glm::vec4 coord_3d = glm::vec4(
                image.pixels[pixel_index * 4],
                image.pixels[pixel_index * 4 + 1],
                image.pixels[pixel_index * 4 + 2],
                image.pixels[pixel_index * 4 + 3]);
            glm::vec4 coord_2d = cam * coord_3d;
            coord_2d = coord_2d / coord_2d[3]; // -1 < x < 1, -1 < y < 1

            max_depth = coord_2d[2] > max_depth ? coord_2d[2] : max_depth;
            min_depth = coord_2d[2] < min_depth ? coord_2d[2] : min_depth;

            camera_coords[pixel_index * 4 + 0] = coord_2d[0];
            camera_coords[pixel_index * 4 + 1] = coord_2d[1];
            camera_coords[pixel_index * 4 + 2] = coord_2d[2];
            camera_coords[pixel_index * 4 + 3] = coord_2d[3];
        }
    }
    myfile << "Image " << index << ": max_depth = " << max_depth << 
        ", min_depth = " << min_depth << std::endl;
    float *depth = new float[image.height * image.width];
    for (int y = 0; y < image.height; y++)
    {
        for (int x = 0; x < image.width; x++)
        {
            int pixel_index = y * image.width + x;
            depth[pixel_index] = (camera_coords[pixel_index * 4 + 2] - min_depth) / (max_depth - min_depth);
        }
    }
    cv::Mat img_depth = cv::Mat(image.height, image.width, CV_32F, depth);
    img_depth.convertTo(img_depth, CV_32F, 255.0);
    std::string output_depth_name = depth_subdir + "\\" + std::to_string(index) + ".png";
    imwrite(output_depth_name, img_depth);
    delete camera_coords;
    delete depth;
    delete image.pixels;
}
myfile.close();
return 0;

} `

[DQSSSSS]

Hi, I modified my code according to yours, but I didn't find the bug. My output of min_dep and max_dep are 0.117861, 0.789501 respectively, but your dataset shows 1.00787 and 1.03434. Maybe your input function 'LoadEXR()' does some preprocessing? I didn't find the file 'EXRImage.h' in some usual libraries, so I'm not sure. Here are my code and result of index=0:

#include <cstdio>
#include <cstdlib>
#include <vector>
#include <cstring>
#include <iostream>
#include <string>

#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include "opencv2/imgproc/imgproc.hpp"

#define TINYEXR_IMPLEMENTATION
#include "tinyexr.h"

#include "sponza/inputs/camera_matrices.h"
const std::string scene = "sponza";

const int frame_size = 1;
const std::string root_path = "E:/proj/data/";
const std::string depth_path = "depth_normalized/";
const std::string worldPosition_path = "inputs/";

void work(std::string scene_path, int index) {
    float* img;
    int width = 0;
    int height = 0;
    const char* err;

    std::string name = (root_path + scene_path +
        worldPosition_path + "world_position" + std::to_string(index) + ".exr");
    LoadEXR(&img, &width, &height, name.c_str(), &err);

    float* img_trans = new float[4 * width * height];
    float* depth = new float[width * height];
    float max_dep = INT_MIN;
    float min_dep = INT_MAX;

    int tot = 0;
    for (int j = 0; j < height; j++) {
        for (int i = 0; i < width; i++) {
            int id = 4 * (j * width + i);

            float ans[4] = {0, 0, 0, 0};
            for (int x = 0; x < 4; x++) {
                for (int y = 0; y < 1; y++) {
                    for (int k = 0; k < 4; k++) {
                        ans[x] += camera_matrices[index][x][k] * img[id + k];
                    }
                }
            }

            for (int k = 0; k < 4; k++) 
                ans[k] /= ans[3], img_trans[id+k] = ans[k];

            depth[id / 4] = ans[2];
            max_dep = max_dep < ans[2] ? ans[2] : max_dep;
            min_dep = min_dep > ans[2] ? ans[2] : min_dep;

        }
    }
    std::cout << min_dep << " " << max_dep << std::endl;

    for (int j = 0; j < height; j++) {
        for (int i = 0; i < width; i++) {
            int id = j * width + i;
            depth[id] = (depth[id] - min_dep) / (max_dep - min_dep);
        }
    }
    cv::Mat img_depth = cv::Mat(height, width, CV_32F, depth);
    img_depth.convertTo(img_depth, CV_32F, 255.0);
    std::string output_depth_name = "result" + std::to_string(index) + ".png";
    imwrite(output_depth_name, img_depth);
    delete img;
    delete img_trans;
    delete depth;
    return;
}

int main() {
    for (int i = 0; i < frame_size; i++) {
        work(scene + "/", i);
    }
}