When using RTSP source, on-screen-display and window-sink operation stops. (not sync)

[YoungjaeDev]

Code content: Code that receives pgie as tensor-meta and draws bbox on osd (later Add align - sgie process). In order to get_nvds_buf_surface in the code, the buffer must be RGBA, so set the source_output_buffer_format like this.

problem: In case of file-source, osd-sink is updated, but in case of rtsp-source, only the first frame appears and the buffer is not synchronized thereafter. However, when I recorded the FPS, the current 15 FPS was displayed well. In other words, there is no major problem, but it seems that sync is not working well on the rendering side.

#!/usr/bin/env python

import os, sys
import io
import yaml

sys.path.append('../dsl')
from dsl import *
import pyds

# 전역 변수로 설정
global config

def load_config():
    global config
    with open('config_face_recognize.yml', 'r') as file:
        config = yaml.safe_load(file)

def get_label_names_from_file(filepath):
    """ Read a label file and convert it to string list """
    f = io.open(filepath, "r")
    labels = f.readlines()
    labels = [elm[:-1] for elm in labels]
    f.close()
    return labels

def add_obj_meta_to_frame(frame_object, batch_meta, frame_meta, label_names, unique_id, scale_x, scale_y, maintain_aspect_ratio, left_padding, top_padding):
    obj_meta = pyds.nvds_acquire_obj_meta_from_pool(batch_meta)

    # Aspect ratio 조정
    if maintain_aspect_ratio:
        scale = min(scale_x, scale_y)
        scale_x, scale_y = scale, scale
    else:
        scale_x, scale_y = scale_x, scale_y

    rect_params = obj_meta.rect_params
    rect_params.left = (frame_object.left - left_padding) * scale_x
    rect_params.top = (frame_object.top - top_padding) * scale_y
    rect_params.width = frame_object.width * scale_x
    rect_params.height = frame_object.height * scale_y

    obj_meta.confidence = frame_object.detectionConfidence
    obj_meta.class_id = frame_object.classId

    # Semi-transparent yellow background
    rect_params.has_bg_color = 0
    rect_params.bg_color.set(1, 1, 0, 0.4)

    # Red border of width 5
    rect_params.border_width = 5
    rect_params.border_color.set(1, 0, 0, 1)

    obj_meta.object_id = config['primary_gie']['untracked_object_id']
    obj_meta.unique_component_id = unique_id

    lbl_id = frame_object.classId
    if lbl_id >= len(label_names):
        raise ValueError("Label ID {} is out of range".format(lbl_id))

    # Set the object classification label.
    obj_meta.obj_label = label_names[lbl_id]

    txt_params = obj_meta.text_params
    if txt_params.display_text:
        pyds.free_buffer(txt_params.display_text)

    txt_params.x_offset = int(rect_params.left)
    txt_params.y_offset = max(0, int(rect_params.top) - 10)
    txt_params.display_text = (
        label_names[lbl_id] + " " + "{:04.3f}".format(frame_object.detectionConfidence)
    )
    # Font , font-color and font-size
    txt_params.font_params.font_name = "Serif"
    txt_params.font_params.font_size = 10
    # set(red, green, blue, alpha); set to White
    txt_params.font_params.font_color.set(1.0, 1.0, 1.0, 1.0)

    # Text background color
    txt_params.set_bg_clr = 1
    # set(red, green, blue, alpha); set to Black
    txt_params.text_bg_clr.set(0.0, 0.0, 0.0, 1.0)

    # Insert the object into current frame meta
    # This object has no parent
    pyds.nvds_add_obj_meta_to_frame(frame_meta, obj_meta, None)

def face_align():
    pass

def layer_finder(output_layer_info, name):
    """ Return the layer contained in output_layer_info which corresponds
        to the given name.
    """
    for layer in output_layer_info:
        # dataType == 0 <=> dataType == FLOAT
        if layer.layerName == name:
            return layer
    return None

def calculate_padding(original_width, original_height, model_width, model_height):
    if (model_width / original_width) > (model_height / original_height):
        scale = model_width / original_width
        adjusted_width = original_width * scale
        total_padding = model_width - adjusted_width
        left_padding = total_padding / 2

        return left_padding, 0
    else:
        scale = model_height / original_height
        # 조정된 높이를 계산합니다.
        adjusted_height = original_height * scale
        total_padding = model_height - adjusted_height
        top_padding = total_padding / 2

        return 0, top_padding

def pgie_src_pad_buffer_probe(buffer, user_data):
    # Retrieve batch metadata from the gst_buffer
    # Note that pyds.gst_buffer_get_nvds_batch_meta() expects the
    # C address of gst_buffer as input, which is obtained with hash(gst_buffer)
    batch_meta = pyds.gst_buffer_get_nvds_batch_meta(buffer)
    l_frame = batch_meta.frame_meta_list

    label_names = get_label_names_from_file(config['primary_gie']['label_file_path'])

    while l_frame is not None:
        try:
            # Note that l_frame.data needs a cast to pyds.NvDsFrameMeta
            # The casting also keeps ownership of the underlying memory
            # in the C code, so the Python garbage collector will leave
            # it alone.
            frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)
        except StopIteration:
            break

        l_user = frame_meta.frame_user_meta_list

        in_surf = pyds.get_nvds_buf_surface(buffer, frame_meta.batch_id)
        in_surf_height, in_surf_width = in_surf.shape[:2]

        while l_user is not None:
            try:
                # Note that l_user.data needs a cast to pyds.NvDsUserMeta
                # The casting also keeps ownership of the underlying memory
                # in the C code, so the Python garbage collector will leave
                # it alone.
                user_meta = pyds.NvDsUserMeta.cast(l_user.data)
            except StopIteration:
                break

            if (user_meta.base_meta.meta_type != pyds.NvDsMetaType.NVDSINFER_TENSOR_OUTPUT_META):
                continue

            tensor_meta = pyds.NvDsInferTensorMeta.cast(user_meta.user_meta_data)

            # Boxes in the tensor meta should be in network resolution which is
            # found in tensor_meta.network_info. Use this info to scale boxes to
            # the input frame resolution.
            layers_info = []

            # https://docs.nvidia.com/metropolis/deepstream/sdk-api/structNvDsInferTensorMeta.html
            unique_id = tensor_meta.unique_id
            netW = tensor_meta.network_info.width
            netH = tensor_meta.network_info.height

            """
            0   INPUT  kFLOAT images          3x704x1280      
            1   OUTPUT kINT32 num_dets        1               
            2   OUTPUT kFLOAT det_boxes       100x4           
            3   OUTPUT kFLOAT det_scores      100             
            4   OUTPUT kINT32 det_classes     100             
            5   OUTPUT kFLOAT det_lmks        100x10          
            6   OUTPUT kFLOAT det_lmks_mask   100x5
            """

            for i in range(tensor_meta.num_output_layers):
                layer = pyds.get_nvds_LayerInfo(tensor_meta, i)
                layers_info.append(layer)
                # https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/blob/8178b5e611ccdc23bef39caf2f1f07b14d39a7cb/bindings/src/bindfunctions.cpp#L767
                # https://docs.nvidia.com/metropolis/deepstream/6.0/sdk-api/structNvDsInferLayerInfo.html

                '''
                layerName:  num_dets
                layerName:  det_boxes
                layerName:  det_scores
                layerName:  det_classes
                layerName:  det_lmks
                layerName:  det_lmks_mask
                '''
                # print("layerName: ", layer.layerName)

                """
                # if layer.layerName == "boxes":
                    # 예시 버퍼 값 출력하기

                    '''
                    m.def("get_detections",
                        [](void *data, int i) {
                            const auto *detections = (const float *) (data);
                            return detections[i];
                        },
                        py::return_value_policy::reference);
                    '''
                    # for j in range(8400):
                        # 모델 사이즈 기준의 Cx, Cy, W, H
                        # print(pyds.get_detections(layer.buffer, j), pyds.get_detections(layer.buffer, j+1), pyds.get_detections(layer.buffer, j+2), pyds.get_detections(layer.buffer, j+3))
                """

            num_detection_layer = layer_finder(layers_info, "num_dets") # num_dets은 int cast 해야함
            box_layer = layer_finder(layers_info, "det_boxes")
            score_layer = layer_finder(layers_info, "det_scores")
            class_layer = layer_finder(layers_info, "det_classes")

            maintain_aspect_ratio = config['primary_gie']['maintain_aspect_ratio']
            symmetric_padding = config['primary_gie']['symmetric_padding']
            left_padding, top_padding = 0, 0

            if symmetric_padding:
                left_padding, top_padding = calculate_padding(in_surf_width, in_surf_height, netW, netH)

            scale_x = in_surf_width / netW
            scale_y = in_surf_height / netH

            for i in range(3):
                ...

                add_obj_meta_to_frame(frame_object, batch_meta, frame_meta, label_names, unique_id, scale_x, scale_y, maintain_aspect_ratio, left_padding, top_padding)

            try:
                l_user = l_user.next
            except StopIteration:
                break

        try:
            # indicate inference is performed on the frame
            frame_meta.bInferDone = True
            l_frame = l_frame.next
        except StopIteration:
            break

    return DSL_PAD_PROBE_OK

def main(args):

    # Since we're not using args, we can Let DSL initialize GST on first call
    while True:

        retval = dsl_pph_custom_new('primary-gie-face-align', pgie_src_pad_buffer_probe, None)
        if retval != DSL_RETURN_SUCCESS:
            break

        # New URI File Source using the filespec defined above
        retval = dsl_source_file_new('file-source', config['source']['file_path'], True)
        if retval != DSL_RETURN_SUCCESS:
            break

        # New RTSP Source
        retval = dsl_source_rtsp_new('rtsp-source',     
            uri = config['source']['rtsp_path'],     
            protocol = DSL_RTP_ALL,         
            skip_frames = 0,     
            drop_frame_interval = 0,     
            latency = config['source']['rtsp_latency'],
            timeout = config['source']['rtsp_timeout'])    
        if retval != DSL_RETURN_SUCCESS:
            break

        retval = dsl_source_video_buffer_out_format_set('file-source', DSL_VIDEO_FORMAT_RGBA)
        if retval != DSL_RETURN_SUCCESS:
            break

        retval = dsl_source_video_buffer_out_format_set('rtsp-source', DSL_VIDEO_FORMAT_RGBA)
        if retval != DSL_RETURN_SUCCESS:
            break

        # 만일, Engine 파일이 없다면 
        if not os.path.isfile(config['primary_gie']['model_engine']):
            retval = dsl_infer_gie_primary_new('primary-gie', 
                config['primary_gie']['infer_config_file'], None, 0)
        else:
            retval = dsl_infer_gie_primary_new('primary-gie', 
                config['primary_gie']['infer_config_file'], config['primary_gie']['model_engine'], 0)
        if retval != DSL_RETURN_SUCCESS:
            break

        # **** IMPORTANT! for best performace we explicity set the GIE's batch-size 
        # to the number of ROI's defined in the Preprocessor configuraton file.
        retval = dsl_infer_batch_size_set('primary-gie', 1)
        if retval != DSL_RETURN_SUCCESS:
            break

        retval = dsl_infer_primary_pph_add('primary-gie', 'primary-gie-face-align', DSL_PAD_SRC)
        if retval != DSL_RETURN_SUCCESS:
            break

        # New IOU Tracker, setting operational frame width and height.
        retval = dsl_tracker_new('nv-tracker', config['tracker']['tracker_config_file'], config['tracker']['width'], config['tracker']['height'])
        if retval != DSL_RETURN_SUCCESS:
            break

        # New OSD with text, clock and bbox display all enabled. 
        retval = dsl_osd_new('on-screen-display', 
            text_enabled=True, clock_enabled=True, 
            bbox_enabled=True, mask_enabled=False)
        if retval != DSL_RETURN_SUCCESS:
            break

        # New Window Sink, 0 x/y offsets and same dimensions as Tiled Display
        retval = dsl_sink_window_new('window-sink', 0, 0, config['sink']['width'], config['sink']['height'])
        if retval != DSL_RETURN_SUCCESS:
            break

        # Add the XWindow event handler functions defined above
        retval = dsl_sink_window_key_event_handler_add("window-sink", 
            xwindow_key_event_handler, None)
        if retval != DSL_RETURN_SUCCESS:
            break
        retval = dsl_sink_window_delete_event_handler_add("window-sink", 
            xwindow_delete_event_handler, None)
        if retval != DSL_RETURN_SUCCESS:
            break

        retval = dsl_sink_sync_enabled_set('window-sink', True)
        if retval != DSL_RETURN_SUCCESS:
            break

        # Add all the components to our pipeline
        retval = dsl_pipeline_new_component_add_many('pipeline', components=[
            'rtsp-source', 'primary-gie', #'nv-tracker', 
            'on-screen-display', 'window-sink', None])
        if retval != DSL_RETURN_SUCCESS:
            break

        # https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_FAQ.html#what-are-different-memory-types-supported-on-jetson-and-dgpu
        # https://github.com/prominenceai/deepstream-services-library/blob/master/docs/api-component.md#dsl_component_nvbuf_mem_type_get
        # retval = dsl_component_nvbuf_mem_type_set('rtsp-source', DSL_NVBUF_MEM_TYPE_CUDA_UNIFIED)
        # if retval != DSL_RETURN_SUCCESS:
        #     break

        # retval = dsl_pipeline_streammux_nvbuf_mem_type_set('pipeline', DSL_NVBUF_MEM_TYPE_CUDA_UNIFIED)
        # if retval != DSL_RETURN_SUCCESS:
        #     break

        ## Add the listener callback functions defined above
        retval = dsl_pipeline_state_change_listener_add('pipeline', state_change_listener, None)
        if retval != DSL_RETURN_SUCCESS:
            break
        retval = dsl_pipeline_eos_listener_add('pipeline', eos_event_listener, None)
        if retval != DSL_RETURN_SUCCESS:
            break

        # Play the pipeline
        retval = dsl_pipeline_play('pipeline')
        if retval != DSL_RETURN_SUCCESS:
            break

        dsl_main_loop_run()
        retval = DSL_RETURN_SUCCESS
        break

    # Print out the final result
    print(dsl_return_value_to_string(retval))

    dsl_pipeline_delete_all()
    dsl_component_delete_all()

if __name__ == '__main__':
    load_config()
    sys.exit(main(sys.argv))

[YoungjaeDev]

I found a solution, but it seems async has become false with the recent dsl update. But could it be a problem in the future whether it works or not depending on async? As far as I know, it is related to sgie, but I am trying to connect sgie separately, and window-sink is also needed.

retval = dsl_sink_async_enabled_set('window-sink', True)
if retval != DSL_RETURN_SUCCESS:
    break

[HoangTienDuc]

Hi @youngjae-avikus, Have you tried to convert RGBA frame to get RGB frame?

           try:
                frame_meta = pyds.NvDsFrameMeta.cast(l_frame.data)

            except StopIteration:
                break
            n_frame = pyds.get_nvds_buf_surface(gst_buffer, frame_meta.batch_id)
            in_surf_height, in_surf_width = n_frame.shape[:2]
            frame_copy = np.array(n_frame, copy=True, order='C')
            rgb_frame = cv2.cvtColor(frame_copy, cv2.COLOR_RGBA2BGR)
            print(n_frame.shape)

I tried it in different ways, but it is failed. If you are success with it, can you pls share it with me?