advanced解码MJPEG，怎样设置输入packet

TRYOKETHEPEN commented 1 month ago

问题： 使用mpp解码UVC视频流，格式为MJPEG，只能使用advanced接口(见https://github.com/rockchip-linux/mpp/issues/586 )。例程mpp/test/mpi_dec_test.c中只有读取二进制文件的示例： https://github.com/rockchip-linux/mpp/blob/develop/test/mpi_dec_test.c#L278

请问读取视频流帧数据应该怎样设置packet？我能拿到的是帧数据的pkt_data、pkt_size，类似 https://github.com/rockchip-linux/rknn-toolkit2/blob/master/rknpu2/examples/rknn_yolov5_demo/utils/mpp_decoder.cpp#L119

分析及尝试： （1）直接mpp_packet_init_with_buffer(&packet, (MppBuffer)pkt_data)，将pkt_data强制转换为MppBuffer来设置packet。会在https://github.com/rockchip-linux/mpp/blob/develop/mpp/base/mpp_buffer_impl.cpp#L170 处出现段错误。并且syslog报错mpp_buffer: group 1296911693 buffer 1296911693 fd 16843013 ops buf ref inc ref_count 263174 caller mpp_packet_init_with_buffer

根据https://github.com/rockchip-linux/mpp/blob/develop/test/mpi_dec_test.c#L94 以及https://github.com/rockchip-linux/rknn-toolkit2/blob/master/rknpu2/examples/rknn_yolov5_demo/utils/mpp_decoder.cpp#L139 可知pkt_data对应的是slot->data，而不是slot->buf 所以不能直接mpp_packet_init_with_buffer(&packet, (MppBuffer)pkt_data)来设置packet。

（2）尝试按照simple接口的packet设置写法：

 mpp_packet_init(&packet, NULL, 0);
 mpp_packet_set_data(packet, pkt_data);
 mpp_packet_set_size(packet, pkt_size);
 mpp_packet_set_pos(packet, pkt_data);
 mpp_packet_set_length(packet, pkt_size);

代码运行到mpi->poll(ctx, MPP_PORT_OUTPUT, MPP_POLL_BLOCK)时，syslog出现报错：

mpp_dec: mpp_dec_advanced_thread line(1072): Error! Get no buffer from input packet
mpp_buffer: check buffer found NULL pointer from mpp_dec_advanced_thread

相关代码： 参考https://github.com/rockchip-linux/rknn-toolkit2/blob/master/rknpu2/examples/rknn_yolov5_demo/utils/mpp_decoder.cpp 的写法，UVC视频流获取pkt_data的部分见https://github.com/rockchip-linux/mpp/issues/586

int MppDecoder::Init(MppCodingType video_type, int fps, int width, int height, void *userdata)
{
    MPP_RET ret = MPP_OK;
    this->last_frame_time_ms = 0;
    this->mpp_type = video_type;
    this->fps = fps;
    this->width_mpp = width;
    this->height_mpp = height;
    this->userdata = userdata;

    memset(&(this->loop_data), 0, sizeof(this->loop_data));
    LOGD("mpi_dec_test decoder test start mpp_type=%d,fps=%d \n", this->mpp_type, this->fps);

    // 创建mpp
    // MppCtx mpp_ctx = NULL;
    ret = mpp_create(&mpp_ctx, &mpp_mpi);
    if (MPP_OK != ret)
    {
        LOGD("mpp_create failed ");
        return 0;
    }

    ret = mpp_init(mpp_ctx, MPP_CTX_DEC, this->mpp_type);
    if (ret != MPP_OK)
    {
        LOGD("%p mpp_init failed ", mpp_ctx);
        return -1;
    }

    // mpp 配置
    MppDecCfg cfg = NULL;
    mpp_dec_cfg_init(&cfg);
    // 加载默认配置
    ret = mpp_mpi->control(mpp_ctx, MPP_DEC_GET_CFG, cfg);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to get decoder cfg ret %d ", mpp_ctx, ret);
        return -1;
    }
    // 设置内部分帧
    ret = mpp_dec_cfg_set_u32(cfg, "base:split_parse", this->need_split);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to set split_parse ret %d ", mpp_ctx, ret);
        return -1;
    }
    // 配置生效
    ret = mpp_mpi->control(mpp_ctx, MPP_DEC_SET_CFG, cfg);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to set cfg %p ret %d ", mpp_ctx, cfg, ret);
        return -1;
    }
    // 释放配置内存
    mpp_dec_cfg_deinit(cfg);

    loop_data.ctx = mpp_ctx;
    loop_data.mpi = mpp_mpi;
    loop_data.eos = 0;
    loop_data.packet_size = this->packet_size;
    loop_data.frame = 0;
    loop_data.frame_count = 0;
    return 1;
}

int MppDecoder::Decode_advanced(uint8_t *pkt_data, int pkt_size, int pkt_eos) //! 对应mpi_dec_test.c中的advanced接口
{
    MPP_RET ret = MPP_OK;
    MpiDecLoopData *data = &loop_data; 
    MppCtx ctx = data->ctx;
    MppApi *mpi = data->mpi;

    //! 设置输出frame的buffer
    MppBuffer frm_buf = NULL;
    ret = dec_buf_mgr_init(&(data->buf_mgr));
    if (ret != MPP_OK)
    {
        LOGD("dec_buf_mgr_init failed\n");
    }
    ret = mpp_frame_init(&frame); /* output frame */
    if (ret != MPP_OK)
    {
        LOGD("mpp_frame_init failed\n");
    }
    data->frm_grp = dec_buf_mgr_setup(data->buf_mgr, MPP_ALIGN(width_mpp, 16) * MPP_ALIGN(height_mpp, 16) * 4, 4, MPP_DEC_BUF_HALF_INT);
    if (!(data->frm_grp))
    {
        LOGD("failed to get buffer group for input frame ret %d\n", ret);
        ret = MPP_NOK;
    }
    ret = mpp_buffer_get(data->frm_grp, &frm_buf, MPP_ALIGN(width_mpp, 16) * MPP_ALIGN(height_mpp, 16) * 4);
    if (ret != MPP_OK)
    {
        mpp_err("failed to get buffer for input frame ret %d\n", ret);
    }
    mpp_frame_set_buffer(frame, frm_buf);

    //! 初始化输入packet
    ret = mpp_packet_init_with_buffer(&packet, (MppBuffer)pkt_data); 
    // ret = mpp_packet_init(&packet, NULL, 0); 
    // mpp_packet_set_data(packet, pkt_data);
    // mpp_packet_set_size(packet, pkt_size);
    // mpp_packet_set_pos(packet, pkt_data);
    // mpp_packet_set_length(packet, pkt_size);

    MppTask task = NULL;
    ret = mpi->poll(ctx, MPP_PORT_INPUT, MPP_POLL_BLOCK);
    if (ret != MPP_OK)
    {
        mpp_err("%p mpp input poll failed\n", ctx);
        return ret;
    }
    ret = mpi->dequeue(ctx, MPP_PORT_INPUT, &task); /* input queue */
    if (ret != MPP_OK)
    {
        mpp_err("%p mpp task input dequeue failed\n", ctx);
        return ret;
    }
    mpp_assert(task);

    ret = mpp_task_meta_set_packet(task, KEY_INPUT_PACKET, packet); 
    ret = mpp_task_meta_set_frame(task, KEY_OUTPUT_FRAME, frame); 

    ret = mpi->enqueue(ctx, MPP_PORT_INPUT, task); /* input queue */
    if (ret != MPP_OK)
    {
        mpp_err("%p mpp task input enqueue failed\n", ctx);
        return ret;
    }
    ret = mpi->poll(ctx, MPP_PORT_OUTPUT, MPP_POLL_BLOCK); /* poll and wait here */
    if (ret != MPP_OK)
    {
        mpp_err("%p mpp output poll failed\n", ctx);
        return ret;
    }
    ret = mpi->dequeue(ctx, MPP_PORT_OUTPUT, &task); /* output queue */
    if (ret != MPP_OK)
    {
        mpp_err("%p mpp task output dequeue failed\n", ctx);
        return ret;
    }
    mpp_assert(task);

    if (task)
    {
        MppFrame frame_out = NULL;
        mpp_task_meta_get_frame(task, KEY_OUTPUT_FRAME, &frame_out);

        if (frame)
        {
            if (callback != nullptr) //! 如果用户设置了解码完成的回调
            {
                RK_U32 hor_stride = mpp_frame_get_hor_stride(frame); // 获取frame信息
                RK_U32 ver_stride = mpp_frame_get_ver_stride(frame);
                RK_U32 hor_width = mpp_frame_get_width(frame);
                RK_U32 ver_height = mpp_frame_get_height(frame);
                MppFrameFormat format = mpp_frame_get_fmt(frame);                         // 获取解码输出帧格式
                char *data_vir = (char *)mpp_buffer_get_ptr(mpp_frame_get_buffer(frame)); // 获取解码输出帧地址
                int fd = mpp_buffer_get_fd(mpp_frame_get_buffer(frame));
                callback(this->userdata, hor_stride, ver_stride, hor_width, ver_height, format, fd, data_vir); //! 调用callback
            }
        }

        ret = mpi->enqueue(ctx, MPP_PORT_OUTPUT, task); /* output queue */
        if (ret != MPP_OK)
            mpp_err("%p mpp task output enqueue failed\n", ctx);
    }

    mpp_packet_deinit(&packet); // 释放packet占用内存

    return ret;
}

HermanChen commented 1 month ago

应该还是用 1 的方式来处理，mpi_dec_test 解码 jpeg 本身能正常么？

TRYOKETHEPEN commented 1 month ago

应该还是用 1 的方式来处理，mpi_dec_test 解码 jpeg 本身能正常么？

mpi_dec_test例程解码jpeg文件，保存为本地raw yuv文件，通过YUView查看是正常的。

用1的方式的话，slot->buf和slot->data没区别吗： https://github.com/rockchip-linux/mpp/blob/4cc3fb25f72fd862596743778575b1aae5b2e9aa/utils/mpi_dec_utils.h#L96

另外，我补充了一些syslog信息。

TRYOKETHEPEN commented 1 month ago

@alexanderdumas 你好，请问可以分享下你在https://github.com/rockchip-linux/mpp/issues/477#issuecomment-1827256689 中是怎么写的吗

TRYOKETHEPEN commented 1 month ago

找到了，参考以下： https://github.com/rockchip-linux/mpp/blob/4cc3fb25f72fd862596743778575b1aae5b2e9aa/mpp/legacy/vpu_api_legacy.cpp#L698 https://github.com/rockchip-linux/mpp/blob/4cc3fb25f72fd862596743778575b1aae5b2e9aa/mpp/legacy/vpu_api_legacy.cpp#L731 https://github.com/rockchip-linux/mpp/blob/4cc3fb25f72fd862596743778575b1aae5b2e9aa/mpp/legacy/vpu_api_legacy.cpp#L736 https://github.com/rockchip-linux/mpp/blob/4cc3fb25f72fd862596743778575b1aae5b2e9aa/mpp/legacy/vpu_api_legacy.cpp#L767

TRYOKETHEPEN commented 1 month ago

共享最终的advanced接口应用代码如下，分为对象初始化Init以及每帧Decode. 参考https://github.com/rockchip-linux/rknn-toolkit2/blob/master/rknpu2/examples/rknn_yolov5_demo/utils/mpp_decoder.cpp

int MppDecoder::Init(MppCodingType video_type, int fps, int width, int height, void *userdata, bool isAdvanced)
{
    MPP_RET ret = MPP_OK;
    this->last_frame_time_ms = 0;
    this->mpp_type = video_type;
    this->fps = fps;
    this->width_mpp = width;
    this->height_mpp = height;
    this->userdata = userdata;

    memset(&(this->loop_data), 0, sizeof(this->loop_data));
    LOGD("mpi_dec_test decoder test start mpp_type=%d,fps=%d \n", this->mpp_type, this->fps);

    // 创建mpp
    ret = mpp_create(&mpp_ctx, &mpp_mpi);
    if (MPP_OK != ret)
    {
        LOGD("mpp_create failed ");
        return 0;
    }
    ret = mpp_init(mpp_ctx, MPP_CTX_DEC, this->mpp_type);
    if (ret != MPP_OK)
    {
        LOGD("%p mpp_init failed ", mpp_ctx);
        return -1;
    }

    // mpp 配置
    MppDecCfg cfg = NULL;
    mpp_dec_cfg_init(&cfg);
    // 加载默认配置
    ret = mpp_mpi->control(mpp_ctx, MPP_DEC_GET_CFG, cfg);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to get decoder cfg ret %d ", mpp_ctx, ret);
        return -1;
    }

    // 设置内部分帧
    ret = mpp_dec_cfg_set_u32(cfg, "base:split_parse", this->need_split);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to set split_parse ret %d ", mpp_ctx, ret);
        return -1;
    }

    // 设置输出帧格式
    MppFrameFormat dstFormat = MPP_FMT_YUV420SP;
    ret = mpp_mpi->control(mpp_ctx, MPP_DEC_SET_OUTPUT_FORMAT, &(dstFormat));
    if (ret != MPP_OK)
    {
        LOGD("Failed to set output format 0x%x\n", dstFormat);
        return -1;
    }

    // 使配置生效
    ret = mpp_mpi->control(mpp_ctx, MPP_DEC_SET_CFG, cfg);
    if (ret != MPP_OK)
    {
        LOGD("%p failed to set cfg %p ret %d ", mpp_ctx, cfg, ret);
        return -1;
    }

    // 释放配置内存
    mpp_dec_cfg_deinit(cfg);

    if (isAdvanced)
    {
        //! 设置输出frame的buffer
        MppBuffer frm_buf = NULL;
        ret = dec_buf_mgr_init(&(loop_data.buf_mgr));
        if (ret != MPP_OK)
        {
            LOGD("dec_buf_mgr_init failed\n");
        }
        ret = mpp_frame_init(&frame); /* output frame */
        if (ret != MPP_OK)
        {
            LOGD("mpp_frame_init failed\n");
        }
        loop_data.frm_grp = dec_buf_mgr_setup(loop_data.buf_mgr, MPP_ALIGN(width_mpp, 16) * MPP_ALIGN(height_mpp, 16) * 4, 4, MPP_DEC_BUF_HALF_INT);
        if (!(loop_data.frm_grp))
        {
            LOGD("failed to get buffer group for input frame ret %d\n", ret);
            ret = MPP_NOK;
        }
        ret = mpp_buffer_get(loop_data.frm_grp, &frm_buf, MPP_ALIGN(width_mpp, 16) * MPP_ALIGN(height_mpp, 16) * 4);
        if (ret != MPP_OK)
        {
            LOGD("failed to get buffer for input frame ret %d\n", ret);
        }
        mpp_frame_set_buffer(frame, frm_buf);
    }

    loop_data.ctx = mpp_ctx;
    loop_data.mpi = mpp_mpi;
    loop_data.eos = 0;
    loop_data.packet_size = this->packet_size;
    loop_data.frame = 0;
    loop_data.frame_count = 0;
    return 1;
}

int MppDecoder::Decode_advanced(uint8_t *pkt_data, int pkt_size, int pkt_eos) //! 对应mpi_dec_test.c中的advanced接口
{
    MPP_RET ret = MPP_OK;
    MpiDecLoopData *data = &loop_data;
    MppCtx ctx = data->ctx;
    MppApi *mpi = data->mpi;

    //! 设置输入packet
    MppBuffer input_buf = NULL;
    ret = mpp_buffer_get(data->pkt_grp, &input_buf, pkt_size);
    if (ret != MPP_OK)
    {
        LOGD("allocate input picture buffer failed\n");
    }
    memcpy((RK_U8 *)mpp_buffer_get_ptr(input_buf), pkt_data, pkt_size);
    ret = mpp_packet_init_with_buffer(&packet, input_buf);

    //! 使用mpptask进行解码
    MppTask task = NULL;
    ret = mpi->poll(ctx, MPP_PORT_INPUT, MPP_POLL_BLOCK); // 查询INPUT端口 是否有数据可供出列  //阻塞
    if (ret != MPP_OK)
    {
        LOGD("%p mpp input poll failed\n", ctx);
        return ret;
    }
    ret = mpi->dequeue(ctx, MPP_PORT_INPUT, &task); // INPUT端口 task出列
    if (ret != MPP_OK)
    {
        LOGD("%p mpp task input dequeue failed\n", ctx);
        return ret;
    }
    // mpp_assert(task);

    ret = mpp_task_meta_set_packet(task, KEY_INPUT_PACKET, packet); // 将packet赋给task
    ret = mpp_task_meta_set_frame(task, KEY_OUTPUT_FRAME, frame);   // 将frame赋给task

    ret = mpi->enqueue(ctx, MPP_PORT_INPUT, task); // INPUT端口 task入列
    if (ret != MPP_OK)
    {
        LOGD("%p mpp task input enqueue failed\n", ctx);
        return ret;
    }
    ret = mpi->poll(ctx, MPP_PORT_OUTPUT, MPP_POLL_BLOCK); // 查询OUTPUT端口 是否有数据可供出列  //阻塞
    if (ret != MPP_OK)
    {
        LOGD("%p mpp output poll failed\n", ctx);
        return ret;
    }
    ret = mpi->dequeue(ctx, MPP_PORT_OUTPUT, &task); // OUTPUT端口 task出列
    if (ret != MPP_OK)
    {
        LOGD("%p mpp task output dequeue failed\n", ctx);
        return ret;
    }
    // mpp_assert(task);

    if (task) // 如果出列的task不为空
    {
        if (frame)
        {
            if (callback != nullptr) // 如果用户设置了解码完成的回调
            {
                RK_U32 hor_stride = mpp_frame_get_hor_stride(frame); // 获取frame信息
                RK_U32 ver_stride = mpp_frame_get_ver_stride(frame);
                RK_U32 hor_width = mpp_frame_get_width(frame);
                RK_U32 ver_height = mpp_frame_get_height(frame);
                MppFrameFormat format = mpp_frame_get_fmt(frame);                         // 获取解码输出帧格式
                char *data_vir = (char *)mpp_buffer_get_ptr(mpp_frame_get_buffer(frame)); // 获取解码输出帧地址
                int fd = mpp_buffer_get_fd(mpp_frame_get_buffer(frame));
                callback(this->userdata, hor_stride, ver_stride, hor_width, ver_height, format, fd, data_vir); //! 调用callback
            }
        }

        ret = mpi->enqueue(ctx, MPP_PORT_OUTPUT, task); /* output queue */
        if (ret != MPP_OK)
        {
            LOGD("%p mpp task output enqueue failed\n", ctx);
        }
    }

    if (packet)
    {
        mpp_packet_deinit(&packet); // 释放packet占用内存
    }
    if (input_buf)
    {
        mpp_buffer_put(input_buf); // 释放buf1占用内存
        input_buf = NULL;
    }

    return ret;
}

rockchip-linux / mpp

advanced解码MJPEG，怎样设置输入packet #592