Upper-Body Detection Image don't image received

[AlessandroMelino]

Hello everyone.

I am trying to use the example of detection and tracking with an Intel Realsense D435, executing the following command, as explain in tutorial 1:

roslaunch spencer_people_tracking_launch tracking_single_rgbd_sensor.launch

Everything seems to be ok but in rviz, the topic /spencer/perception_internal/people_detection/rgbd_front_top/upper_body_detector/image set the warning No image received and nothing is detected.

What am I doing wrong?

I post the whole roslaunch code that I am using:

<launch>
    <!-- Launch file arguments -->
    <arg name="height_above_ground" default="0.96"/>  <!-- in meters, assumes a horizontally oriented RGB-D sensor; important for accurate detection -->
    <arg name="load_driver" default="false"/>  <!-- set to false if you are already running OpenNi from elsewhere -->
    <arg name="load_realsense_driver" default="true"/>  <!-- set to false if you are already running OpenNi from elsewhere -->
    <arg name="visualization" default="true"/> <!-- start Rviz with predefined configuration? -->
    <arg name="dummy_transforms" default="true"/> <!-- publish TF transform for sensor to groundplane -->
    <arg name="use_upper_body_detector" default="true"/> <!-- use depth-template-based upper-body detector? -->
    <arg name="use_pcl_detector" default="false"/> <!-- use ROI-HOG detector from PCL? Cannot be combined with upper-body detector! -->
    <arg name="use_hog_detector" default="false"/> <!-- use RGB groundHOG detector? Requires cudaHOG library-->

    <!-- TF frame IDs used by some detectors, and tracking components -->
    <arg name="base_footprint_frame_id" default="base_footprint"/>  <!-- name of the robot's base frame after projection onto the ground plane -->
    <arg name="world_frame_id" default="odom"/>  <!-- this is the fixed tracking frame -->

    <!--Model of the wheelchair. -->

    <param name="robot_description" textfile="$(find kate_global)/urdf/kate_model.urdf"/>
    <node name="robot_state_publisher" pkg="robot_state_publisher" type="robot_state_publisher" />

    <!-- Run OpenNi driver if desired -->
    <group ns="spencer/sensors" if="$(arg load_driver)">
        <arg name="driver_launchfile" value="$(find openni2_launch)/launch/openni2.launch"/>  <!-- Realsense-->

        <include file="$(arg driver_launchfile)">
          <arg name="camera" value="rgbd"/>
          <arg name="device_id" value="#1"/>
          <arg name="depth_registration" value="true"/>
        </include>
    </group>

    <!-- Run Realsense driver if desired -->
    <group ns="spencer/sensors" if="$(arg load_realsense_driver)">
        <arg name="driver_launchfile" value="$(find realsense2_camera)/launch/rs_rgbd.launch"/>  <!-- Realsense-->

        <include file="$(arg driver_launchfile)">
          <!--arg name="camera" value="rgbd"/-->
          <!--arg name="device_id" value="#1"/-->
          <!--arg name="depth_registration" value="true"/-->
          <arg name="camera" value="rgbd_front_top"/>
        </include>
    </group>

    <!-- Set ground plane distance -->
    <rosparam param="/spencer/perception_internal/people_detection/ground_plane/distance" subst_value="true">$(arg height_above_ground)</rosparam>

    <!-- Set up dummy transforms into an imaginary robot and odom frame -->
    <group if="$(arg dummy_transforms)">
        <node name="tf_base_footprint" pkg="tf" type="static_transform_publisher" args="0 0 $(arg height_above_ground) 0 0 0 $(arg base_footprint_frame_id) rgbd_front_top_link 10"/>
        <node name="tf_odom"           pkg="tf" type="static_transform_publisher" args="0 0 0 0 0 0 odom $(arg base_footprint_frame_id) 10"/>
        <node name="tf_base"           pkg="tf" type="static_transform_publisher" args="0 0 0 0 0 0 odom base_link 10"/>
    </group>

    <!-- Detectors -->
    <group>
        <include file="$(find spencer_people_tracking_launch)/launch/detectors/front_rgbd_detectors.launch">
            <arg name="upper_body" default="$(arg use_upper_body_detector)"/>
            <arg name="pcl_detector" value="$(arg use_pcl_detector)"/>
            <arg name="hog" default="$(arg use_hog_detector)"/>
            <arg name="base_footprint_frame_id" value="$(arg base_footprint_frame_id)"/>
        </include>
    </group>

    <!-- People tracking -->
    <include file="$(find spencer_people_tracking_launch)/launch/tracking/freiburg_people_tracking.launch">
        <arg name="rgbd" default="true"/>
        <arg name="laser_low_confidence_detections" default="false"/>
        <arg name="base_footprint_frame_id" value="$(arg base_footprint_frame_id)"/>
        <arg name="world_frame_id" value="$(arg world_frame_id)"/>
    </include>

    <!-- As there is not yet any high-recall/low-confidence detector for RGB-D, and we are not using laser, tracks may get deleted too quickly in case of missed detections.
         To deal with this, for the moment, we increase the maximum number of occluded frames to be  a bit more tolerant towards missed detections.
         This works fine in uncrowded environments which are not very dynamic. -->
    <rosparam param="/spencer/perception_internal/people_tracking/srl_nearest_neighbor_tracker/max_occlusions_before_deletion">50</rosparam>
    <rosparam param="/spencer/perception_internal/people_tracking/srl_nearest_neighbor_tracker/max_occlusions_before_deletion_of_mature_track">200</rosparam>

    <!-- Group tracking -->
    <include file="$(find spencer_people_tracking_launch)/launch/tracking/group_tracking.launch"/>

    <!-- RViz visualization-->
    <node name="tracking_visualization_rviz" pkg="rviz" type="rviz" args="-d $(find spencer_people_tracking_launch)/rviz/tracking-single-realsense.rviz" if="$(arg visualization)"/>

</launch>

EDIT: I have tested just the Upper Body Detector through a modified launch file of roslaunch rwth_upper_body_detector upper_body_detector.launch. The file is the following:

<?xml version="1.0"?>
<launch>
    <!-- detector config -->
    <arg name="load_params_from_file" default="true" />
    <arg name="config_file" default="$(find rwth_upper_body_detector)/config/upper_body_detector_realsense.yaml" />
    <arg name="template_file" default="$(find rwth_upper_body_detector)/config/upper_body_template.yaml" />
    <arg name="height_above_ground" default="0.96"/>  <!-- in meters, assumes a horizontally oriented RGB-D sensor; important for accurate detection -->

    <arg name="queue_size" default="5" />
    <arg name="camera_namespace" default="/rgbd_realsense" />

    <!-- input topic names, can either be configured via these args or via remappings -->
    <arg name="depth_image" default="/depth/image_rect_raw" />
    <arg name="rgb_image" default="/color/image_raw" />
    <arg name="camera_info_depth" default="/depth/camera_info" />

    <!-- output topic names, can either be configured via these args or via remappings -->
    <arg name="upper_body_detections" default="/upper_body_detector/detections" />
    <arg name="upper_body_bb_centres" default="/upper_body_detector/bounding_box_centres" />
    <arg name="upper_body_image" default="/upper_body_detector/image" />
    <arg name="upper_body_markers" default="/upper_body_detector/marker_array" />
    <arg name="upper_body_roi" default="/upper_body_detector/roi" />
    <arg name="upper_body_closest_bb_centres" default="/upper_body_detector/closest_bounding_box_centre" />

    <!-- SPENCER integration -->
    <arg name="detected_persons" default="/detected_persons"/>
    <arg name="detection_id_offset" default="0"/>
    <arg name="detection_id_increment" default="1"/>

    <!-- Ground Plane configuration -->
    <arg name="param_file" default="$(find rwth_ground_plane)/config/fixed_gp.yaml" />
    <arg name="ptu_state" default="/ptu/state" />
    <arg name="ground_plane" default="/ground_plane" />

    <!-- TF frame IDs used by some detectors, and tracking components -->
    <arg name="base_footprint_frame_id" default="base_footprint"/>  <!-- name of the robot's base frame after projection onto the ground plane -->
    <arg name="world_frame_id" default="odom"/>  <!-- this is the fixed tracking frame -->

    <!--Model of the wheelchair. -->
    <param name="robot_description" textfile="$(find kate_global)/urdf/kate_model.urdf"/>
    <node name="robot_state_publisher" pkg="robot_state_publisher" type="robot_state_publisher" />

    <rosparam command="load" file="$(arg config_file)" if="$(arg load_params_from_file)"/>
    <rosparam command="load" file="$(arg template_file)" if="$(arg load_params_from_file)"/>

    <!-- Run Realsense -->
    <arg name="driver_launchfile" value="$(find realsense2_camera)/launch/rs_rgbd.launch"/>
    <include file="$(arg driver_launchfile)">
      <arg name="camera" value="rgbd_realsense"/>
    </include>

    <!-- Run ground_plane -->
    <node pkg="rwth_ground_plane" type="ground_plane_fixed" name="ground_plane" output="screen">
        <rosparam command="load" file="$(arg param_file)"/>
        <param name="ptu_state" value="$(arg ptu_state)" type="string"/>
        <param name="ground_plane" value="$(arg ground_plane)" type="string"/>
    </node>

    <node name="tf_base_footprint" pkg="tf" type="static_transform_publisher" args="0 0 $(arg height_above_ground) 0 0 0 $(arg base_footprint_frame_id) rgbd_realsense_link 10"/>
    <node name="tf_odom"           pkg="tf" type="static_transform_publisher" args="0 0 0 0 0 0 odom $(arg base_footprint_frame_id) 10"/>
    <node name="tf_base"           pkg="tf" type="static_transform_publisher" args="0 0 0 0 0 0 odom base_link 10"/>

    <!-- Detectors -->
    <node pkg="rwth_upper_body_detector" type="upper_body_detector" name="upper_body_detector" output="screen" respawn="true">
        <param name="queue_size" value="$(arg queue_size)" type="int"/>
        <param name="camera_namespace" value="$(arg camera_namespace)" type="string"/>

        <param name="ground_plane" value="$(arg ground_plane)" type="string"/>
        <param name="depth_image" value="$(arg depth_image)" type="string"/>
        <param name="rgb_image" value="$(arg rgb_image)" type="string"/>
        <param name="camera_info_depth" value="$(arg camera_info_depth)" type="string"/>

        <param name="upper_body_detections" value="$(arg upper_body_detections)" type="string"/>
        <param name="upper_body_bb_centres" value="$(arg upper_body_bb_centres)" type="string"/>
        <param name="upper_body_image" value="$(arg upper_body_image)" type="string"/>
        <param name="upper_body_markers" value="$(arg upper_body_markers)" type="string"/>
        <param name="upper_body_roi" value="$(arg upper_body_roi)" type="string"/>
        <param name="upper_body_closest_bb_centres" value="$(arg upper_body_closest_bb_centres)" type="string"/>

        <param name="detected_persons" value="$(arg detected_persons)" type="string"/>
        <param name="detection_id_offset" value="$(arg detection_id_offset)"/>
        <param name="detection_id_increment" value="$(arg detection_id_increment)"/>
    </node>

    <!-- RViz visualization-->
    <node name="tracking_visualization_rviz" pkg="rviz" type="rviz" args="-d $(find spencer_people_tracking_launch)/rviz/detector-realsense.rviz"/>

</launch>

The problem is the same, no people detected and no image received on the detector.

Thanks in advance.

Best regards. Alessandro

[AlessandroMelino]

Finally solved remapping the topic, changing the measure of the depth_image (640x480) and changing the depth escale to 1000 in de configuration.yaml file.

[frequiem11]

Could you please give elaborated details? How does your configuration.yaml file look like?

[AlessandroMelino]

Yes of course @frequiem11 , this is my .launch file:

<?xml version="1.0"?>

<launch>
    <!-- detector config -->
    <arg name="load_params_from_file" default="true" />
    <arg name="config_file" default="$(find rwth_upper_body_detector)/config/upper_body_detector_realsense.yaml" />
    <arg name="template_file" default="$(find rwth_upper_body_detector)/config/upper_body_template.yaml" />

    <arg name="queue_size" default="5" />
    <arg name="camera_namespace" default="/head_xtion" />

    <!-- input topic names, can either be configured via these args or via remappings -->
    <arg name="ground_plane" default="/ground_plane" />
    <arg name="depth_image" default="/depth/image_rect_raw" />
    <arg name="rgb_image" default="/rgb/image_rect_color" />
    <arg name="camera_info_depth" default="/depth/camera_info" />

    <!-- output topic names, can either be configured via these args or via remappings -->
    <arg name="upper_body_detections" default="/upper_body_detector/detections" />
    <arg name="upper_body_bb_centres" default="/upper_body_detector/bounding_box_centres" />
    <arg name="upper_body_image" default="/upper_body_detector/image" />
    <arg name="upper_body_markers" default="/upper_body_detector/marker_array" />
    <arg name="upper_body_roi" default="/upper_body_detector/roi" />
    <arg name="upper_body_closest_bb_centres" default="/upper_body_detector/closest_bounding_box_centre" />

    <!-- SPENCER integration -->
    <arg name="detected_persons" default="/detected_persons"/>
    <arg name="detection_id_offset" default="0"/>
    <arg name="detection_id_increment" default="1"/>

    <rosparam command="load" file="$(arg config_file)" if="$(arg load_params_from_file)"/>
    <rosparam command="load" file="$(arg template_file)" if="$(arg load_params_from_file)"/>

    <node pkg="rwth_upper_body_detector" type="upper_body_detector" name="upper_body_detector" output="screen" respawn="true">
        <param name="queue_size" value="$(arg queue_size)" type="int"/>
        <param name="camera_namespace" value="$(arg camera_namespace)" type="string"/>

        <param name="ground_plane" value="$(arg ground_plane)" type="string"/>
        <param name="depth_image" value="$(arg depth_image)" type="string"/>
        <param name="rgb_image" value="$(arg rgb_image)" type="string"/>
        <param name="camera_info_depth" value="$(arg camera_info_depth)" type="string"/>

        <param name="upper_body_detections" value="$(arg upper_body_detections)" type="string"/>
        <param name="upper_body_bb_centres" value="$(arg upper_body_bb_centres)" type="string"/>
        <param name="upper_body_image" value="$(arg upper_body_image)" type="string"/>
        <param name="upper_body_markers" value="$(arg upper_body_markers)" type="string"/>
        <param name="upper_body_roi" value="$(arg upper_body_roi)" type="string"/>
        <param name="upper_body_closest_bb_centres" value="$(arg upper_body_closest_bb_centres)" type="string"/>

        <param name="detected_persons" value="$(arg detected_persons)" type="string"/>
        <param name="detection_id_offset" value="$(arg detection_id_offset)"/>
        <param name="detection_id_increment" value="$(arg detection_id_increment)"/>
    </node>

</launch>

and my .yaml file:

#=====================================
# Upper body detector parameters
#=====================================
upper_body_detector:
    #=====================================
    # Distance Range Accepted Detections
    #=====================================
    distance_range_accepted_detections: 7 # up to what distance detections are kept. All detections after that distance are rejected

    #=====================================
    # ROI
    #=====================================
    inc_width_ratio: 0.2
    inc_height_ratio: 0.0
    region_size_threshold: 10

    #======================
    # Freespace
    #======================
    # parameteters for occupancy Map computation

    # these parameter define the number of bins for the occupancy histogram
    freespace_scaleZ: 20
    freespace_scaleX: 20

    freespace_minX: -3
    freespace_minZ: 0
    freespace_maxX: 3
    freespace_maxZ: 7
    freespace_threshold: 500
    freespace_max_depth_to_cons: 7

    #====================================
    # Detector
    #====================================
    # parameters upper body detector
    evaluation_NMS_threshold_LM: 0.25 # lowering this parameter will reduce the number of false positives! Keep in mind this is distance!
    evaluation_inc_height_ratio: 0.2  # parameter for increasing the size of ROI which is then scanned by the depth template

    # <<< these parameters are only relevant if you want to perform multi-scaling, evaluation_nr_scales > 1
    evaluation_stride: 3
    evaluation_scale_stride: 1.03
    evaluation_nr_scales: 1
    evaluation_inc_cropped_height: 20
    evaluation_greedy_NMS_overlap_threshold: 0.3
    evaluation_greedy_NMS_threshold: 0.25
    # >>>

    # scanning corridor, which means for each ROI we scan it with the depth template starting from min_height (in meters) for up to max_height
    max_height: 2.1
    min_height: 1.0

    #======================
    # World scale
    #======================
    WORLD_SCALE: 1.0

    #======================
    # Depth scale
    #======================
    DEPTH_SCALE: 1000.0

    #===========================
    # height and width of images
    #===========================
    dImWidth: 640
    dImHeight: 480

    #====================================
    # Number of Frames / offset
    #====================================
    numberFrames: 1000000
    nOffset: 0

    #====================================
    # Size of Template
    #====================================
    template_size: 30

By the way, it works but no very efficiently.

Best regards. Alessandro

[Taysssir]

Hello, I'm using ifm3d sensor and I faced same issue. I fix height and width of images and setting Depth scale to 1000 but still getting the warning : No image received and nothing is detected. Any idea How to solve it please ?