How would one receive a RGB image?

[j-friedl]

Hello again, I was trying out some python samples (like these) and came to a problem when trying to get RGB images out of my O3R Vision System. Following code was executed:

from ifm3dpy import O3RCamera, ImageBuffer, FrameGrabber
import matplotlib.pyplot as plt

# Initialize the objects
o3r = O3RCamera()
fg = FrameGrabber(o3r, pcic_port=50010) #Expecting a head on Port 0 (Port 0 == 50010)
im = ImageBuffer()

# Get a frame
if fg.wait_for_frame(im, 500) == False:
    raise ValueError #Exception('fg-timeout on ' + port + ' reached')

plt.imshow(im.jpeg_image())
plt.show()

However, wait_for_frame returns false and I get the risen ValueError:

Traceback (most recent call last): File "main.py", line 11, in raise ValueError #Exception('fg-timeout on ' + port + ' reached') ValueError

It does work with port 50012 and im.depth_image(). All the ports are in RUN mode. What is up here?

[j-friedl]

The output of ifm3d dump:

{
  "device": {
    "clock": {
      "currentTime": 1581097822432306400
    },
    "diagnostic": {
      "temperatures": [
        {
          "entity": "PLL-therm",
          "value": 42.5
        },
        {
          "entity": "MCPU-therm",
          "value": 42.5
        },
        {
          "entity": "PMIC-Die",
          "value": 100.0
        },
        {
          "entity": "Tboard_tegra",
          "value": 39.0
        },
        {
          "entity": "GPU-therm",
          "value": 40.0
        },
        {
          "entity": "BCPU-therm",
          "value": 42.5
        },
        {
          "entity": "thermal-fan-est",
          "value": 41.29999923706055
        },
        {
          "entity": "Tdiode_tegra",
          "value": 39.25
        },
        {
          "entity": "port2",
          "value": 62
        }
      ],
      "upTime": 7178000000000
    },
    "info": {
      "description": "",
      "device": "0302",
      "deviceTreeBinaryBlob": "tegra186-quill-p3310-1000-c03-00-base.dtb",
      "guiSettings": "",
      "name": "my_new_o3r",
      "partNumber": "OVP800",
      "productionState": "AA",
      "serialNumber": "000018429580",
      "vendor": "0001"
    },
    "log": {
      "level": "warn",
      "storage": "volatile"
    },
    "network": {
      "authorized_keys": "removed for this post",
      "interfaces": {
        "eth0": {
          "ipAddressConfig": 0,
          "ipv4": {
            "address": "192.168.0.69",
            "dns": "192.168.0.255",
            "gateway": "192.168.0.201",
            "mask": 24
          },
          "mac": "48:B0:2D:54:F2:36",
          "networkSpeed": 100,
          "useDHCP": false
        },
        "eth1": {
          "ipAddressConfig": 1,
          "mac": "00:02:01:43:28:FF",
          "networkSpeed": 10,
          "useDHCP": true
        }
      }
    },
    "state": {
      "errorMessage": "",
      "errorNumber": ""
    },
    "swVersion": {
      "euphrates": "1.4.3",
      "firmware": "0.14.23-493",
      "kernel": "4.9.140-l4t-r32.4+g1582a8a5405d",
      "l4t": "r32.4.3",
      "schema": "v0.2.4",
      "tcu": "1.0.2"
    }
  },
  "ports": {
    "port0": {
      "acquisition": {
        "framerate": 10.0,
        "version": {
          "major": 0,
          "minor": 0,
          "patch": 0
        }
      },
      "data": {
        "algoDebugConfig": {
          "channels": [
            {
              "id": "ov9782/ifcustom",
              "type": "normal"
            },
            {
              "id": "ov9782/profile",
              "type": "normal"
            },
            {
              "id": "ov9782/dbg",
              "type": "normal"
            },
            {
              "id": "ov9782/calib",
              "type": "once"
            },
            {
              "id": "ov9782/ifout",
              "type": "normal"
            },
            {
              "id": "ov9782/frameInfo",
              "type": "normal"
            }
          ],
          "version": {
            "major": 0,
            "minor": 11,
            "patch": 12
          }
        },
        "availablePCICOutput": [
          "RGB_INFO"
        ],
        "pcicTCPPort": 50010
      },
      "info": {
        "calibration": {
          "version": {
            "major": 0,
            "minor": 1,
            "patch": 1
          }
        },
        "device": "2301",
        "deviceTreeBinaryBlobOverlay": "001-ov9782.dtbo",
        "features": {
          "fov": {
            "horizontal": 127,
            "vertical": 80
          },
          "resolution": {
            "height": 800,
            "width": 1280
          },
          "type": "2D"
        },
        "icc": {
          "version": {
            "major": 0,
            "minor": 0,
            "patch": 0
          }
        },
        "name": "",
        "partNumber": "O3R225",
        "productionState": "AA",
        "sensor": "OV9782",
        "sensorID": "OV9782_127x80_noIllu_Csample",
        "serialNumber": "000018565855",
        "vendor": "0001",
        "version": {
          "major": 11,
          "minor": 1,
          "patch": 0
        }
      },
      "mode": "experimental_autoexposure2D",
      "processing": {
        "extrinsicHeadToUser": {
          "rotX": 0.0,
          "rotY": 0.0,
          "rotZ": 0.0,
          "transX": 0.0,
          "transY": 0.0,
          "transZ": 0.0
        },
        "version": {
          "major": 0,
          "minor": 11,
          "patch": 12
        }
      },
      "state": "RUN"
    },
    "port2": {
      "acquisition": {
        "channelSelection": "manual",
        "channelValue": 0,
        "exposureLong": 5000,
        "exposureShort": 400,
        "framerate": 8,
        "offset": 0.0,
        "version": {
          "major": 0,
          "minor": 0,
          "patch": 0
        }
      },
      "data": {
        "algoDebugConfig": {
          "channels": [
            {
              "id": "irs2381/ifcustom",
              "type": "normal"
            },
            {
              "id": "irs2381/profile",
              "type": "normal"
            },
            {
              "id": "irs2381/dbg",
              "type": "normal"
            },
            {
              "id": "irs2381/calib",
              "type": "once"
            },
            {
              "id": "irs2381/ifout_compr",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/mode",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/0",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/1",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/2",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/3",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/4",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/5",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/6",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/7",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/8",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/9",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/10",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/11",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/12",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/13",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/14",
              "type": "normal"
            },
            {
              "id": "irs2381/raw/15",
              "type": "normal"
            }
          ],
          "version": {
            "major": 0,
            "minor": 11,
            "patch": 12
          }
        },
        "availablePCICOutput": [
          "TOF_INFO",
          "RADIAL_DISTANCE_NOISE",
          "RADIAL_DISTANCE_COMPRESSED",
          "REFLECTIVITY",
          "AMPLITUDE_COMPRESSED",
          "CONFIDENCE"
        ],
        "pcicTCPPort": 50012
      },
      "info": {
        "calibration": {
          "version": {
            "major": 0,
            "minor": 5,
            "patch": 1
          }
        },
        "device": "3101",
        "deviceTreeBinaryBlobOverlay": "001-irs2381c.dtbo",
        "features": {
          "fov": {
            "horizontal": 105,
            "vertical": 78
          },
          "resolution": {
            "height": 172,
            "width": 224
          },
          "type": "3D"
        },
        "icc": {
          "version": {
            "major": 1,
            "minor": 0,
            "patch": 4
          }
        },
        "name": "",
        "partNumber": "O3R225",
        "productionState": "AA",
        "sensor": "IRS2381C",
        "sensorID": "IRS2381C_105x78_4x2W_110x90_C7",
        "serialNumber": "000018565855",
        "vendor": "0001",
        "version": {
          "major": 11,
          "minor": 1,
          "patch": 0
        }
      },
      "mode": "standard_range4m",
      "processing": {
        "diParam": {
          "anfFilterSizeDiv2": 2,
          "dynamicSymmetryThreshold": 20.0,
          "enableTemporalFilter": true,
          "maxDistNoise": 0.05,
          "medianSizeDiv2": 0,
          "minAmplitude": 20.0,
          "minReflectivity": 2.0,
          "mixedPixelFilterMode": 1,
          "mixedPixelThresholdRad": 0.05
        },
        "extrinsicHeadToUser": {
          "rotX": 0.0,
          "rotY": 0.0,
          "rotZ": 0.0,
          "transX": 0.0,
          "transY": 0.0,
          "transZ": 0.0
        },
        "version": {
          "major": 0,
          "minor": 11,
          "patch": 12
        }
      },
      "state": "RUN"
    }
  }
}

[desengph]

@j-friedl can you please double-check that you can get any 2D RGB Images on port0 via the ifm Vision Assistant. The ifm Vision Assistant can be downloaded here. This version is compatible with the O3R in a development stage.

If you can get data here, we need to look into this in a deeper level. Alternatively, a power-cycle might solve such issues already.

[lola-masson]

Additionnally, there is a known issue in firmware 14.23 (which is the latest release) where you need to switch the rgb cameras to CONF and then back to RUN for them to stream data. Try that and see if that helps.

[j-friedl]

It kinda works when changing from CONF to RUN, but the image I receive is now 125067 by 1 in size (changing slightly every time I rerun). How is the image encoded and/or how can I convert it?

[inbangsa]

@j-friedl The data you receive is jpeg encoded blob and not RGB image, you need to decode the blob and convert the blob to 3channel RGB image.

with opencv you can try like this:

image = cv2.imdecode(im.jpeg_image(), cv2.cv::IMREAD_UNCHANGED)

[j-friedl]

@inbangsa thanks for the suggestion, this led me on the right track. My solution is as follows:

#include <ifm3d/camera/camera_o3r.h>
#include <ifm3d/fg.h>
#include <ifm3d/stlimage.h>
#include <ifm3d/stlimage/image.h>

#include <opencv2/opencv.hpp>
#include <opencv2/core/mat.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>

#include <cstdint>
#include <vector>

// OpenCV to Ifm3d converter  
std::map<uint32_t, int32_t> ifm3d_type_to_cv_type{
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_8U),  CV_8U},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_8S),  CV_8S},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_16U), CV_16U},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_16S), CV_16S},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_32S), CV_32S},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_32F), CV_32F},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_64F), CV_64F},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_16U2),CV_16U},
  {static_cast<std::uint32_t>(ifm3d::pixel_format::FORMAT_32F3),CV_32F}};

namespace ifm3d
{

  cv::Mat
  convert_to_mat(ifm3d::Image& image)
  {
    int cv_type = CV_MAKETYPE(
      ifm3d_type_to_cv_type[static_cast<std::uint32_t>(image.dataFormat())],
      image.nchannels());

    return cv::Mat(image.height(), image.width(), cv_type, (void*)image.ptr(0));
  }
}

int main(){
    // Declare the device object (one object only, corresponding to the VPU)
    auto cam = std::make_shared<ifm3d::O3RCamera>();
    // Declare the FrameGrabber and ImageBuffer objects. 
    const auto FG_PCIC_PORT = cam->Get()["/ports/port0/data/pcicTCPPort"_json_pointer];
    auto fg = std::make_shared<ifm3d::FrameGrabber>(cam, ifm3d::DEFAULT_SCHEMA_MASK, FG_PCIC_PORT);
    auto im =  std::make_shared<ifm3d::StlImageBuffer>();

    if (! fg->WaitForFrame(im.get(), 3000))
    {
      std::cerr << "Timeout waiting for camera!" << std::endl;
      return -1;
    }

    ifm3d::Image ifm3d_img = im->JPEGImage();
    cv::Mat cv_mat = ifm3d::convert_to_mat(ifm3d_img);

    if(cv_mat.empty())
    {
        std::cout << "Could not read the image" << std::endl;
        return 1;
    }

    // for RGB image:
    std::vector<uchar> vec = cv_mat;
    cv::Mat rgb_mat = cv::imdecode(vec, cv::IMREAD_ANYCOLOR);
    cv::imshow("ifm3d",rgb_mat);
    cv::waitKey(0);
    return 0;
}

Using following CMakeLists.txt file:

# Specify the minimum version
cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
# Specify the project name
project(ifm3d-example)
# Declare the project target
add_executable(${PROJECT_NAME}
main.cpp
)
SET("OpenCV_DIR" "/usr/share/OpenCV")
find_package(OpenCV REQUIRED)

find_package(ifm3d REQUIRED)

# Libraries to link
target_link_libraries(${PROJECT_NAME} PRIVATE
                    # ifm3d libs
                     ifm3d_framegrabber
                     ifm3d_camera
                     ifm3d_stlimage
                     /usr/lib/libglog.so.0.3.5 # not sure why this one is needed?
                    # opencv libs
                     ${OpenCV_LIBS}
                     )

This issue can be closed now, thank you!

[inbangsa]

@j-friedl Happy to help,

just to make it short you can try

auto buffer = ifm3d_img.ptr<uint8_t>(0);
auto matImg = cv::imdecode(cv::Mat(1, bufferLength, CV_8UC1, buffer), CV_LOAD_IMAGE_UNCHANGED);

following link will have more inputs.

https://stackoverflow.com/questions/4271489/how-to-use-cvimdecode-if-the-contents-of-an-image-file-are-in-a-char-array

[j-friedl]

@inbangsa Thank you! Works like a charm. Although CV_LOAD_IMAGE_UNCHANGED must be changed to cv::IMREAD_UNCHANGED in newer versions. Like this:

    auto buffer = ifm3d_img.ptr<uint8_t>(0);
    auto matImg = cv::imdecode(cv::Mat(1, ifm3d_img.height() * ifm3d_img.width() * sizeof(uint8_t), CV_8UC1, buffer), cv::IMREAD_UNCHANGED);

[github-actions[bot]]

This issue is stale because it has been open for 30 days with no activity.

[github-actions[bot]]

This issue was closed because it has been inactive for 14 days since being marked as stale.