search

prominenceai / deepstream-services-library

A shared library of on-demand DeepStream Pipeline Services for Python and C/C++
MIT License
273 stars 64 forks source link

DSL src/DslPipelineStateMgr.cpp:472:HandleErrorMessage: : Error message 'Internal data stream error.' received from 'tiler-queue' #1206

Open hadestyz opened 4 months ago

hadestyz commented 4 months ago

Hi : Deepstream 6.4 dsl 0.29 cuda 12,2 driver 535.104.12

When I initialized the pipeline and tried to add a new source to it, the analysis and inference seemed to have stopped. 

  def run(self):
        # Since we're not using args, we can Let DSL initialize GST on first call
        while True:
            retval = self._CreatePerSourceComponents('pipeline', 1, self.video_url, None)
            if retval != DSL_RETURN_SUCCESS:
                break

            # New Primary GIE using the filespecs above, with infer interval
            retval = dsl_infer_gie_primary_new('primary-gie', self.inferConfigFile, self.modelEngineFile, 0)
            if retval != DSL_RETURN_SUCCESS:
                break

            # New IOU Tracker, setting operational width and hieght
            retval = dsl_tracker_new('iou-tracker', self.iou_tracker_config_file, 480, 272)
            if retval != DSL_RETURN_SUCCESS:
                break

            # New Tiler, setting width and height, use default cols/rows set by source count
            retval = dsl_tiler_new('tiler', TILER_WIDTH, TILER_HEIGHT)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_tiler_tiles_set('tiler', columns=2, rows=2)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_pph_custom_new('custom-pph',client_handler=self._save_frame, client_data=None)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_pph_custom_new('custom-pph2',client_handler=self._tiler_sink_pad_buffer_probe, client_data=None)
            if retval != DSL_RETURN_SUCCESS:
                break

            # Add the ODE Pad Probe Hanlder to the Sink Pad of the Tiler
            retval = dsl_tiler_pph_add('tiler', 'custom-pph', DSL_PAD_SINK)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_tiler_pph_add('tiler', 'custom-pph2', DSL_PAD_SINK)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_sink_fake_new('fake-sink')
            if retval != DSL_RETURN_SUCCESS:
                break

            # Add all the components to our pipeline
            retval = dsl_pipeline_new_component_add_many('pipeline',['source-1', 'primary-gie', 'iou-tracker', 'tiler', 'fake-sink', None])
            if retval != DSL_RETURN_SUCCESS:
                break

            ### IMPORTANT: we need to explicitely set the stream-muxer Batch properties, otherwise the Pipeline
            # will use the current number of Sources when set to Playing, which would be 1 and too small
            retval = dsl_pipeline_streammux_batch_properties_set('pipeline', self.MAX_SOURCE_COUNT, 40000)
            # retval = dsl_pipeline_streammux_batch_properties_set('pipeline', 1, 40000)
            if retval != DSL_RETURN_SUCCESS:
                break
            retval = dsl_pipeline_streammux_dimensions_set('pipeline', self.max_width, self.max_height)
            if retval != DSL_RETURN_SUCCESS:
                break

            retval = dsl_pipeline_eos_listener_add('pipeline', self._eos_event_listener, None)

            if retval != DSL_RETURN_SUCCESS:
                break

            # Play the pipeline
            retval = dsl_pipeline_play('pipeline')
            if retval != DSL_RETURN_SUCCESS:
                break

            dsl_main_loop_run()
            retval = DSL_RETURN_SUCCESS
            break

        # Print out the final result
        print(dsl_return_value_to_string(retval))

        dsl_pipeline_delete_all()
        dsl_component_delete_all()
    def _add_source(self, source_id, video_url):
        source_name = 'source-' + str(source_id+1)
        print('adding source ', source_name)

        retval = self._CreatePerSourceComponents("pipeline", source_id+1, video_url, None)
        if retval != DSL_RETURN_SUCCESS:
            return
        retval = dsl_pipeline_component_add('pipeline', source_name)
        if retval != DSL_RETURN_SUCCESS:
            return
        print('add source sucess', source_name)
        return True
    def _stop_source(self, task_data, content=None):
        source_name = 'source-' + str(task_data["source_id"]+1)

        if task_data["source_id"] == 0 
            return

        print('removing source ', source_name)
        retval = dsl_pipeline_component_remove('pipeline', source_name)
        if retval != DSL_RETURN_SUCCESS:
            content = "video component remove error, occur action: {}".format(content)

        retval = dsl_component_delete(source_name)
        if retval != DSL_RETURN_SUCCESS:
            content = "video component delete error, occur action: {}".format(content)

        print('removing source sucess', source_name)
        if content:
            pass
        else:
            pass

        return True

When I initialized the pipeline and tried to add a new source to it, the analysis and inference seemed to have stopped. The log is as follows:

0:00:34.051324824 34038 0x7feb80013920 WARN            v4l2videodec gstv4l2videodec.c:1303:gst_v4l2_video_dec_loop:<nvv4l2decoder1> Decoder is producing too many buffers
0:00:34.051371385 34038 0x7feb80013920 WARN            v4l2videodec gstv4l2videodec.c:1303:gst_v4l2_video_dec_loop:<nvv4l2decoder1> Decoder is producing too many buffers
0:00:34.052060811 34038 0x7feb80013920 WARN            v4l2videodec gstv4l2videodec.c:1303:gst_v4l2_video_dec_loop:<nvv4l2decoder1> Decoder is producing too many buffers
nvstreammux: Successfully handled EOS for source_id=1
adding source  source-3
0:00:45.176445807 34038 0x7febf4080040 WARN                     DSL src/DslSourceBintr.cpp:2021:SetFileUri:  : Unable to determine video frame-rate and dimensions for URI Source = 'source-3' Extended AV File Services are disabled in the Makefile
0:00:45.179371704 34038 0x7febf4080040 WARN                 basesrc gstbasesrc.c:3688:gst_base_src_start_complete:<source> pad not activated yet
0:00:45.180719313 34038 0x7febf4080040 WARN                 basesrc gstbasesrc.c:3688:gst_base_src_start_complete:<source> pad not activated yet
add source sucess source-3
0:00:45.185634486 34038 0x7febd40661e0 WARN                 qtdemux qtdemux.c:3121:qtdemux_parse_trex:<qtdemux2> failed to find fragment defaults for stream 1
0:00:45.190114657 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190158489 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat MJPG
0:00:45.190176711 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190195365 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat MJPG
0:00:45.190238447 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190259247 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat AV10
0:00:45.190283629 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190302914 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat AV10
0:00:45.190331074 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190348536 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat DVX5
0:00:45.190365061 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190382652 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat DVX5
0:00:45.190407831 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190425041 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat DVX4
0:00:45.190446171 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190463596 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat DVX4
0:00:45.190493202 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190512598 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat MPG4
0:00:45.190529210 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190546741 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat MPG4
0:00:45.190574413 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190592670 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat MPG2
0:00:45.190608864 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190627587 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat MPG2
0:00:45.190657312 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190675540 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat H265
0:00:45.190691762 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190709447 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat H265
0:00:45.190735758 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190753881 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat VP90
0:00:45.190770057 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190788400 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat VP90
0:00:45.190813144 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190831170 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat VP80
0:00:45.190846980 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190863777 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat VP80
0:00:45.190890835 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190908603 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe minimum capture size for pixelformat H264
0:00:45.190925116 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:sink> Unable to try format: Unknown error -1
0:00:45.190941141 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:sink> Could not probe maximum capture size for pixelformat H264
0:00:45.191029115 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191049580 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe minimum capture size for pixelformat Y444
0:00:45.191067001 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191085314 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe maximum capture size for pixelformat Y444
0:00:45.191123016 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191142795 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe minimum capture size for pixelformat P410
0:00:45.191159930 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191178459 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe maximum capture size for pixelformat P410
0:00:45.191211540 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191230726 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe minimum capture size for pixelformat PM10
0:00:45.191247153 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191265281 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe maximum capture size for pixelformat PM10
0:00:45.191297752 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191316768 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2985:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe minimum capture size for pixelformat NM12
0:00:45.191333558 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:3100:gst_v4l2_object_get_nearest_size:<nvv4l2decoder2:src> Unable to try format: Unknown error -1
0:00:45.191351380 34038 0x7feb80013920 WARN                    v4l2 gstv4l2object.c:2991:gst_v4l2_object_probe_caps_for_format:<nvv4l2decoder2:src> Could not probe maximum capture size for pixelformat NM12
0:00:45.310299097 34038 0x7feb80013920 WARN            v4l2videodec gstv4l2videodec.c:2258:gst_v4l2_video_dec_decide_allocation:<nvv4l2decoder2> Duration invalid, not setting latency
0:00:45.310804441 34038 0x7feb80013920 WARN          v4l2bufferpool gstv4l2bufferpool.c:1114:gst_v4l2_buffer_pool_start:<nvv4l2decoder2:pool:src> Uncertain or not enough buffers, enabling copy threshold
0:00:45.312356558 34038 0x7feb783d12a0 WARN          v4l2bufferpool gstv4l2bufferpool.c:1565:gst_v4l2_buffer_pool_dqbuf:<nvv4l2decoder2:pool:src> Driver should never set v4l2_buffer.field to ANY
nvstreammux: Successfully handled EOS for source_id=2
removing source  source-2
nvstreammux: Successfully handled EOS for source_id=1
removing source sucess source-2
0:01:00.172307395 34038 0x55f71db67aa0 WARN                   queue gstqueue.c:992:gst_queue_handle_sink_event:<tiler-queue> error: Internal data stream error.
0:01:00.172375276 34038 0x55f71db67aa0 WARN                   queue gstqueue.c:992:gst_queue_handle_sink_event:<tiler-queue> error: streaming stopped, reason error (-5)
0:01:00.172663444 34038 0x55f71d617460 ERROR                    DSL src/DslPipelineStateMgr.cpp:472:HandleErrorMessage:  : Error message 'Internal data stream error.' received from 'tiler-queue'

tiler like this

    def _tiler_sink_pad_buffer_probe(self, buffer, user_data):
        # Retrieve batch metadata from the gst_buffer
        try:
            batch_meta = pyds.gst_buffer_get_nvds_batch_meta(buffer)
            l_frame = batch_meta.frame_meta_list
        except:
            return DSL_PAD_PROBE_OK

        while l_frame is not None:
            try:
                frame_meta = pyds.glist_get_nvds_frame_meta(l_frame.data)
            except StopIteration:
                print('_tiler_sink_pad_buffer_probe error --------------------------------')
                break

            frame_number = frame_meta.frame_num
            source_id = frame_meta.source_id
            l_obj = frame_meta.obj_meta_list

            if source_id != 0:
                boxes = []
                track_boxes = []
                txt_data = []
                while l_obj is not None:
                    try:
                        # Casting l_obj.data to pyds.NvDsObjectMeta
                        obj_meta = pyds.glist_get_nvds_object_meta(l_obj.data)
                    except StopIteration:
                        print('_tiler_sink_pad_buffer_probe error2 --------------------------------')
                        break

                    boxes.append([float(obj_meta.detector_bbox_info.org_bbox_coords.left), float(obj_meta.detector_bbox_info.org_bbox_coords.top),
                                  float(obj_meta.detector_bbox_info.org_bbox_coords.width), float(obj_meta.detector_bbox_info.org_bbox_coords.height),
                                  int(obj_meta.class_id), float(obj_meta.confidence)])
                    track_boxes.append([float(obj_meta.detector_bbox_info.org_bbox_coords.left), float(obj_meta.detector_bbox_info.org_bbox_coords.top),
                                        float(obj_meta.detector_bbox_info.org_bbox_coords.left) + float(obj_meta.detector_bbox_info.org_bbox_coords.width),
                                        float(obj_meta.detector_bbox_info.org_bbox_coords.top) + float(obj_meta.detector_bbox_info.org_bbox_coords.height),
                                        float(obj_meta.confidence), obj_meta.class_id])
                    try:
                        l_obj = l_obj.next
                    except StopIteration:
                        print('_tiler_sink_pad_buffer_probe error3 --------------------------------')
                        break
                if track_boxes and source_id in self.stream_data.keys():

                    _, match_ids = self.stream_data[source_id]["byte_track"].update(np.array(track_boxes))
                    for _, track in enumerate(match_ids):
                        boxes[int(track[-1])].append(int(track[0]))

            if source_id != 0 and frame_number * self.drop_frame_interval % self.frame_skip == 0 and source_id in self.stream_data.keys() and self.stream_data[source_id]["task_id"] in self.img_list_dict.keys():
           pass

            try:
                l_frame = l_frame.next
            except StopIteration:
                print('_tiler_sink_pad_buffer_probe error4 --------------------------------')
                break

        return DSL_PAD_PROBE_OK

Do you have any ideas? Thank you

rjhowell44 commented 4 months ago

@hadestyz I'll be able to look at this tomorrow

hadestyz commented 4 months ago

@rjhowell44 I compared the examples and found the cause of the problem: If I use this, it will result in the above error: retval=dsl_source_uri_new (components. source, uri, False, False, 4) If I use this, it looks like it runs normally: retval=dsl_source_uri_new(components. source, uri, False, 0, 0)