Segfault with denoise image operation

[paulmelis]

// Copyright 2009-2019 Intel Corporation
// SPDX-License-Identifier: Apache-2.0

/* This is a small example tutorial how to use OSPRay in an application.
 *
 * On Linux build it in the build_directory with
 *   g++ ../apps/ospTutorial/ospTutorial.cpp -I ../ospray/include \
 *       -I ../../ospcommon -L . -lospray -Wl,-rpath,. -o ospTutorial
 * On Windows build it in the build_directory\$Configuration with
 *   cl ..\..\apps\ospTutorial\ospTutorial.cpp /EHsc -I ..\..\ospray\include ^
 *      -I ..\.. -I ..\..\..\ospcommon ospray.lib
 */

#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#ifdef _WIN32
#define NOMINMAX
#include <malloc.h>
#else
#include <alloca.h>
#endif

#include <vector>

#include "ospray/ospray_cpp.h"

using namespace ospcommon::math;

// helper function to write the rendered image as PPM file
void writePPM(const char *fileName, const vec2i &size, const uint32_t *pixel)
{
  FILE *file = fopen(fileName, "wb");
  if (file == nullptr) {
    fprintf(stderr, "fopen('%s', 'wb') failed: %d", fileName, errno);
    return;
  }
  fprintf(file, "P6\n%i %i\n255\n", size.x, size.y);
  unsigned char *out = (unsigned char *)alloca(3 * size.x);
  for (int y = 0; y < size.y; y++) {
    const unsigned char *in =
        (const unsigned char *)&pixel[(size.y - 1 - y) * size.x];
    for (int x = 0; x < size.x; x++) {
      out[3 * x + 0] = in[4 * x + 0];
      out[3 * x + 1] = in[4 * x + 1];
      out[3 * x + 2] = in[4 * x + 2];
    }
    fwrite(out, 3 * size.x, sizeof(char), file);
  }
  fprintf(file, "\n");
  fclose(file);
}

int main(int argc, const char **argv)
{
  // image size
  vec2i imgSize;
  imgSize.x = 1024; // width
  imgSize.y = 768; // height

  // camera
  vec3f cam_pos{0.f, 0.f, 0.f};
  vec3f cam_up{0.f, 1.f, 0.f};
  vec3f cam_view{0.1f, 0.f, 1.f};

  // triangle mesh data
  std::vector<vec3f> vertex = {vec3f(-1.0f, -1.0f, 3.0f),
      vec3f(-1.0f, 1.0f, 3.0f),
      vec3f(1.0f, -1.0f, 3.0f),
      vec3f(0.1f, 0.1f, 0.3f)};

  std::vector<vec4f> color = {vec4f(0.9f, 0.5f, 0.5f, 1.0f),
      vec4f(0.8f, 0.8f, 0.8f, 1.0f),
      vec4f(0.8f, 0.8f, 0.8f, 1.0f),
      vec4f(0.5f, 0.9f, 0.5f, 1.0f)};

  std::vector<vec3ui> index = {vec3ui(0, 1, 2), vec3ui(1, 2, 3)};

  ospLoadModule("denoiser");

  // initialize OSPRay; OSPRay parses (and removes) its commandline parameters,
  // e.g. "--osp:debug"
  OSPError init_error = ospInit(&argc, argv);
  if (init_error != OSP_NO_ERROR)
    return init_error;

  // use scoped lifetimes of wrappers to release everything before ospShutdown()
  {
    // create and setup camera
    ospray::cpp::Camera camera("perspective");
    camera.setParam("aspect", imgSize.x / (float)imgSize.y);
    camera.setParam("position", cam_pos);
    camera.setParam("direction", cam_view);
    camera.setParam("up", cam_up);
    camera.commit(); // commit each object to indicate modifications are done

    // create and setup model and mesh
    ospray::cpp::Geometry mesh("mesh");
    mesh.setParam("vertex.position", ospray::cpp::Data(vertex));
    mesh.setParam("vertex.color", ospray::cpp::Data(color));
    mesh.setParam("index", ospray::cpp::Data(index));
    mesh.commit();

    // put the mesh into a model
    ospray::cpp::GeometricModel model(mesh);
    model.commit();

    // put the model into a group (collection of models)
    ospray::cpp::Group group;
    group.setParam("geometry", ospray::cpp::Data(model));
    group.commit();

    // put the group into an instance (give the group a world transform)
    ospray::cpp::Instance instance(group);
    instance.commit();

    // put the instance in the world
    ospray::cpp::World world;
    world.setParam("instance", ospray::cpp::Data(instance));

    // create and setup light for Ambient Occlusion
    ospray::cpp::Light light("ambient");
    light.commit();

    world.setParam("light", ospray::cpp::Data(light));
    world.commit();

    // create renderer, choose Scientific Visualization renderer
    ospray::cpp::Renderer renderer("scivis");

    // complete setup of renderer
    renderer.setParam("aoSamples", 1);
    renderer.setParam("backgroundColor", 1.0f); // white, transparent
    renderer.commit();

    // denoise
    ospray::cpp::ImageOperation denoise("denoiser");
    denoise.commit();

    // create and setup framebuffer
    ospray::cpp::FrameBuffer framebuffer(
        imgSize, OSP_FB_SRGBA, 
        OSP_FB_COLOR | OSP_FB_ACCUM | OSP_FB_ALBEDO | OSP_FB_NORMAL);
    framebuffer.setParam("imageOperation", ospray::cpp::Data(denoise));
    framebuffer.commit();

    framebuffer.clear();

    // render one frame
    framebuffer.renderFrame(renderer, camera, world);

    // access framebuffer and write its content as PPM file
    uint32_t *fb = (uint32_t *)framebuffer.map(OSP_FB_COLOR);
    writePPM("firstFrameCpp.ppm", imgSize, fb);
    framebuffer.unmap(fb);
  }

  ospShutdown();

  return 0;
}

The code itself seems to be ok, if I replace the denoiser imageop with a tonemapper one it works fine. But running the above gives me a segfault, looks like a null pointer:

paulm@cmstorm 09:10:~/concepts/ospray-python/utests$ g++ -o t_denoise -I ~/software/ospray-superbuild-debug-git/include/ -L ~/software/ospray-superbuild-debug-git/lib/ t_denoise.cpp -lospray
paulm@cmstorm 09:10:~/concepts/ospray-python/utests$ gdb ./t_denoise
GNU gdb (GDB) 8.3.1
...
Reading symbols from ./t_denoise...
(No debugging symbols found in ./t_denoise)
(gdb) run
Starting program: /home/paulm/concepts/ospray-python/utests/t_denoise 
/usr/lib/../share/gcc-9.2.0/python/libstdcxx/v6/xmethods.py:731: SyntaxWarning: list indices must be integers or slices, not str; perhaps you missed a comma?
  refcounts = ['_M_refcount']['_M_pi']
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/usr/lib/libthread_db.so.1".
[New Thread 0x7fffe8b57700 (LWP 478315)]
[New Thread 0x7fffe3fff700 (LWP 478317)]
[New Thread 0x7fffe8756700 (LWP 478316)]
[New Thread 0x7fffe3bfe700 (LWP 478318)]
[New Thread 0x7fffe37fd700 (LWP 478319)]
[New Thread 0x7fffe33fc700 (LWP 478320)]
[New Thread 0x7fffe2ffb700 (LWP 478321)]

Thread 1 "t_denoise" received signal SIGSEGV, Segmentation fault.
0x00007ffff432d8c1 in ospray::LocalFrameBuffer::commit (this=0x555555592650) at /home/paulm/c/ospray-git/ospray/fb/LocalFB.cpp:98
98        imageOps.push_back(obj->attach(fbv));
(gdb) p obj
$1 = (ospray::ImageOp *&) @0x7fffe914f880: 0x0

[jeffamstutz]

The denoiser requires using a float frame buffer (OSP_FB_RGBA32F), which ends up triggering an error with any other pixel format. However, because the error isn't caught, undefined behavior is encountered (which results in a segfault).

[paulmelis]

Hmmm, using a OSP_FB_RGBA32F framebuffer (updated test below), I still get the same segfault.

``

include

include

include

ifdef _WIN32

define NOMINMAX

include

else

include

endif

include

include "ospray/ospray_cpp.h"

using namespace ospcommon::math;

// helper function to write the rendered image as PPM file void writePPM(const char fileName, const vec2i &size, const uint32_t pixel) { FILE file = fopen(fileName, "wb"); if (file == nullptr) { fprintf(stderr, "fopen('%s', 'wb') failed: %d", fileName, errno); return; } fprintf(file, "P6\n%i %i\n255\n", size.x, size.y); unsigned char out = (unsigned char )alloca(3 size.x); for (int y = 0; y < size.y; y++) { const unsigned char in = (const unsigned char )&pixel[(size.y - 1 - y) size.x]; for (int x = 0; x < size.x; x++) { out[3 x + 0] = in[4 x + 0]; out[3 x + 1] = in[4 x + 1]; out[3 x + 2] = in[4 x + 2]; } fwrite(out, 3 size.x, sizeof(char), file); } fprintf(file, "\n"); fclose(file); }

int main(int argc, const char **argv) { // image size vec2i imgSize; imgSize.x = 1024; // width imgSize.y = 768; // height

// camera vec3f cam_pos{0.f, 0.f, 0.f}; vec3f cam_up{0.f, 1.f, 0.f}; vec3f cam_view{0.1f, 0.f, 1.f};

// triangle mesh data std::vector vertex = {vec3f(-1.0f, -1.0f, 3.0f), vec3f(-1.0f, 1.0f, 3.0f), vec3f(1.0f, -1.0f, 3.0f), vec3f(0.1f, 0.1f, 0.3f)};

std::vector color = {vec4f(0.9f, 0.5f, 0.5f, 1.0f), vec4f(0.8f, 0.8f, 0.8f, 1.0f), vec4f(0.8f, 0.8f, 0.8f, 1.0f), vec4f(0.5f, 0.9f, 0.5f, 1.0f)};

std::vector index = {vec3ui(0, 1, 2), vec3ui(1, 2, 3)};

ospLoadModule("denoiser");

// initialize OSPRay; OSPRay parses (and removes) its commandline parameters, // e.g. "--osp:debug" OSPError init_error = ospInit(&argc, argv); if (init_error != OSP_NO_ERROR) return init_error;

// use scoped lifetimes of wrappers to release everything before ospShutdown() { // create and setup camera ospray::cpp::Camera camera("perspective"); camera.setParam("aspect", imgSize.x / (float)imgSize.y); camera.setParam("position", cam_pos); camera.setParam("direction", cam_view); camera.setParam("up", cam_up); camera.commit(); // commit each object to indicate modifications are done

// create and setup model and mesh
ospray::cpp::Geometry mesh("mesh");
mesh.setParam("vertex.position", ospray::cpp::Data(vertex));
mesh.setParam("vertex.color", ospray::cpp::Data(color));
mesh.setParam("index", ospray::cpp::Data(index));
mesh.commit();

// put the mesh into a model
ospray::cpp::GeometricModel model(mesh);
model.commit();

// put the model into a group (collection of models)
ospray::cpp::Group group;
group.setParam("geometry", ospray::cpp::Data(model));
group.commit();

// put the group into an instance (give the group a world transform)
ospray::cpp::Instance instance(group);
instance.commit();

// put the instance in the world
ospray::cpp::World world;
world.setParam("instance", ospray::cpp::Data(instance));

// create and setup light for Ambient Occlusion
ospray::cpp::Light light("ambient");
light.commit();

world.setParam("light", ospray::cpp::Data(light));
world.commit();

// create renderer, choose Scientific Visualization renderer
ospray::cpp::Renderer renderer("scivis");

// complete setup of renderer
renderer.setParam("aoSamples", 1);
renderer.setParam("backgroundColor", 1.0f); // white, transparent
renderer.commit();

// denoise
ospray::cpp::ImageOperation denoise("denoiser");
denoise.commit();

// create and setup framebuffer
ospray::cpp::FrameBuffer framebuffer(
    imgSize, OSP_FB_RGBA32F, 
    OSP_FB_COLOR | OSP_FB_ACCUM | OSP_FB_ALBEDO | OSP_FB_NORMAL);
framebuffer.setParam("imageOperation", ospray::cpp::Data(denoise));
framebuffer.commit();

framebuffer.clear();

// render one frame
framebuffer.renderFrame(renderer, camera, world);

uint8_t *colors = new uint8_t[imgSize.x*imgSize.y*4];
float *pixel;

for (int i = 0; i < imgSize.x*imgSize.y; i++)
{
    pixel[3*i+0] = uint8_t(colors[4*i+0] * 255.0f);
    pixel[3*i+1] = uint8_t(colors[4*i+1] * 255.0f);
    pixel[3*i+2] = uint8_t(colors[4*i+2] * 255.0f);
    pixel[3*i+3] = 255;
}

uint32_t *fb = (uint32_t *)framebuffer.map(OSP_FB_COLOR);
framebuffer.unmap(fb);

writePPM("denoised.ppm", imgSize, (uint32_t*)colors);

}

ospShutdown();

return 0; }

[paulmelis]

Oh, looking at the sample, is the image operation called "frame_denoise"?

(Plus, there conversion errors in my test above when saving the image)

[johguenther]

Well, it should be named "denoiser", another oversight...

[jeffamstutz]

We can easily make an alias for the name denoiser and denoise. The current naming convention is forward looking for users to know if it's a pixel/tile/frame operation, as the order of them can have performance consequences (i.e. the whole frame has to finish before a frame operation can proceed).

In other words, we have several other image operation implementations almost ready to go.

[jeffamstutz]

My mistake: apparently we no longer prefix image ops with the type they implement. :)

[paulmelis]

Okay, working now as expected, closing