rk3588 视频编码，编码数据有问题

liwang54321 commented 1 year ago

你好，我用mpp编码hdmi in 进来的BGR888数据，然后通过v4l2 采集视频数据，这部分没有问题，我查看了保存的图片，是正确的。然后到mpp进行数据编码，编码产生的数据有问题，基本上是复用了demo, 不知道哪里配置错了，请教一下大神，能帮忙看一下吗

#pragma once

#include <rockchip/rk_mpi.h>

#include <functional>
#include <string>
#include <thread>

struct FrameInfo {
    uint32_t height;
    uint32_t width;
    std::string format;
    uint8_t fps;
};

struct StreamInfo {
    std::string StreamType;
    uint32_t gop;
};

struct MppEncInfo {
    int32_t width;
    int32_t height;
    int32_t hor_stride;
    int32_t ver_stride;
    int32_t frame_size;
    int32_t header_size;
    int32_t mdinfo_size;
    int32_t bps;
    MppCodingType code_type;
    MppFrameFormat frame_format;

    MppEncRcMode rc_mode;
    // MppBufferGroup buf_grp;
    // MppBuffer frm_buf;
    // MppBuffer pkt_buf;
    // MppBuffer md_info;
    MppEncSeiMode sei_mode;
    MppEncHeaderMode header_mode;
};

struct StreamPackage {
    uint8_t* data;
    uint32_t len;
    bool is_eos;
};

class VideoEncoder {
public:
    VideoEncoder(const FrameInfo& frame_info, const StreamInfo& stream_info,
        int timeout = -1, bool is_camera_dma = false);
    ~VideoEncoder(void);

    bool Init(const std::function<void(uint8_t* data, uint32_t size)>& package_callback);

    bool PutFrame(uint8_t* data, uint32_t size, int dma_fd = -1);

private:
    static void EncRecvThread(VideoEncoder* self);
    bool SetMppEncCfg(void);

private:
    MppCtx ctx_ = nullptr;
    MppApi* api_ = nullptr;
    MppEncCfg cfg_ = nullptr;
    // MppEncRcCfg rc_cfg_;
    // MppEncPrepCfg prep_cfg_;
    // MppEncCodecCfg codec_cfg_;

    int timeout = -1;
    bool is_camera_dma_ = false;
    FrameInfo frame_info_;
    StreamInfo stream_info_;
    MppEncInfo enc_info_;

    bool is_running_ = false;
    std::thread recv_thread_;
    std::function<void(uint8_t* data, uint32_t size)> package_callback_;
};

#include "VideoEncoder.h"

#include <spdlog/spdlog.h>
#include <magic_enum.hpp>

#define MPP_ALIGN(x, a) (((x) + (a)-1) & ~((a)-1))

VideoEncoder::VideoEncoder(const FrameInfo& frame_info, const StreamInfo& stream_info,
    int timeout, bool is_camera_dma)
    : frame_info_(frame_info)
    , stream_info_(stream_info)
    , is_running_(true)
    , is_camera_dma_(is_camera_dma)
{
}

VideoEncoder::~VideoEncoder(void)
{
    if (ctx_ != nullptr) {
        mpp_destroy(ctx_);
    }
}

static inline MppCodingType AdaptStreamType(std::string_view stream_type)
{
    if (stream_type.compare("H265") == 0) {
        return MPP_VIDEO_CodingHEVC;
    } else if (stream_type.compare("H264") == 0) {
        return MPP_VIDEO_CodingAVC;
    }

    spdlog::error("Unkown support type {}", stream_type);
    return MPP_VIDEO_CodingUnused;
}

static inline MppFrameFormat AdaptFrameType(std::string_view stream_type)
{
    if (stream_type.compare("BGR24") == 0) {
        return MPP_FMT_BGR888;
    }

    spdlog::error("Unkonw Frame Type {}", stream_type);
    return MPP_FMT_BUTT;
}
static inline int GetFrameSize(MppFrameFormat frame_format, int32_t hor_stride, int32_t ver_stride)
{
    int32_t frame_size = 0;
    switch (frame_format & MPP_FRAME_FMT_MASK) {
    case MPP_FMT_YUV420SP:
    case MPP_FMT_YUV420P: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 3 / 2;
    } break;

    case MPP_FMT_YUV422_YUYV:
    case MPP_FMT_YUV422_YVYU:
    case MPP_FMT_YUV422_UYVY:
    case MPP_FMT_YUV422_VYUY:
    case MPP_FMT_YUV422P:
    case MPP_FMT_YUV422SP: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 2;
    } break;
    case MPP_FMT_RGB444:
    case MPP_FMT_BGR444:
    case MPP_FMT_RGB555:
    case MPP_FMT_BGR555:
    case MPP_FMT_RGB565:
    case MPP_FMT_BGR565:
    case MPP_FMT_RGB888:
    case MPP_FMT_BGR888:
    case MPP_FMT_RGB101010:
    case MPP_FMT_BGR101010:
    case MPP_FMT_ARGB8888:
    case MPP_FMT_ABGR8888:
    case MPP_FMT_BGRA8888:
    case MPP_FMT_RGBA8888: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64);
    } break;

    default: {
        frame_size = MPP_ALIGN(hor_stride, 64) * MPP_ALIGN(ver_stride, 64) * 4;
    } break;
    }
    return frame_size;
}

#define SZ_1K (1024)
#define SZ_2K (SZ_1K * 2)
#define SZ_4K (SZ_1K * 4)
static inline int GetHeaderSize(MppFrameFormat frame_format, uint32_t width, uint32_t height)
{
    int header_size = 0;
    if (MPP_FRAME_FMT_IS_FBC(frame_format)) {
        if ((frame_format & MPP_FRAME_FBC_MASK) == MPP_FRAME_FBC_AFBC_V1)
            header_size = MPP_ALIGN(MPP_ALIGN(width, 16) * MPP_ALIGN(height, 16) / 16, SZ_4K);
        else
            header_size = MPP_ALIGN(width, 16) * MPP_ALIGN(height, 16) / 16;
    } else {
        header_size = 0;
    }
    return header_size;
}

bool VideoEncoder::SetMppEncCfg(void)
{
    mpp_enc_cfg_set_s32(cfg_, "prep:width", enc_info_.width);
    mpp_enc_cfg_set_s32(cfg_, "prep:height", enc_info_.height);
    mpp_enc_cfg_set_s32(cfg_, "prep:hor_stride", enc_info_.hor_stride);
    mpp_enc_cfg_set_s32(cfg_, "prep:ver_stride", enc_info_.ver_stride);
    mpp_enc_cfg_set_s32(cfg_, "prep:format", enc_info_.frame_format);

    mpp_enc_cfg_set_s32(cfg_, "rc:mode", enc_info_.rc_mode);

    /* fix input / output frame rate */
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_flex", 0);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_num", frame_info_.fps);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_in_denorm", 1);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_flex", 0);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_num", frame_info_.fps);
    mpp_enc_cfg_set_s32(cfg_, "rc:fps_out_denorm", 1);
    mpp_enc_cfg_set_s32(cfg_, "rc:gop", stream_info_.gop ? stream_info_.gop : frame_info_.fps * 2);

    /* drop frame or not when bitrate overflow */
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED);
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_thd", 20); /* 20% of max bps */
    mpp_enc_cfg_set_u32(cfg_, "rc:drop_gap", 1); /* Do not continuous drop frame */

    /* setup bitrate for different rc_mode */
    mpp_enc_cfg_set_s32(cfg_, "rc:bps_target", enc_info_.bps);
    switch (enc_info_.rc_mode) {
    case MPP_ENC_RC_MODE_FIXQP: {
        /* do not setup bitrate on FIXQP mode */
    } break;
    case MPP_ENC_RC_MODE_CBR: {
        /* CBR mode has narrow bound */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 15 / 16);
    } break;
    case MPP_ENC_RC_MODE_VBR:
    case MPP_ENC_RC_MODE_AVBR: {
        /* VBR mode has wide bound */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 1 / 16);
    } break;
    default: {
        /* default use CBR mode */
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_max", enc_info_.bps * 17 / 16);
        mpp_enc_cfg_set_s32(cfg_, "rc:bps_min", enc_info_.bps * 15 / 16);
    } break;
    }

    /* setup qp for different codec and rc_mode */
    switch (enc_info_.code_type) {
    case MPP_VIDEO_CodingAVC:
    case MPP_VIDEO_CodingHEVC: {
        switch (enc_info_.rc_mode) {
        case MPP_ENC_RC_MODE_FIXQP: {
            RK_S32 fix_qp = 0;
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", fix_qp);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 0);
        } break;
        case MPP_ENC_RC_MODE_CBR:
        case MPP_ENC_RC_MODE_VBR:
        case MPP_ENC_RC_MODE_AVBR: {
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", -1);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", 51);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", 10);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", 51);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", 10);
            mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 2);
        } break;
        default: {
            spdlog::error("unsupport encoder rc mode {}", enc_info_.rc_mode);
        } break;
        }
    } break;
    case MPP_VIDEO_CodingVP8: {
        /* vp8 only setup base qp range */
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_init", 40);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_max", 127);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_min", 0);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_max_i", 127);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_min_i", 0);
        mpp_enc_cfg_set_s32(cfg_, "rc:qp_ip", 6);
    } break;
    case MPP_VIDEO_CodingMJPEG: {
        /* jpeg use special codec config to control qtable */
        mpp_enc_cfg_set_s32(cfg_, "jpeg:q_factor", 80);
        mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_max", 99);
        mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_min", 1);
    } break;
    default: {
    } break;
    }

    /* setup codec  */
    mpp_enc_cfg_set_s32(cfg_, "codec:type", enc_info_.code_type);
    switch (enc_info_.code_type) {
    case MPP_VIDEO_CodingAVC: {
        RK_U32 constraint_set;
        /*
         * H.264 profile_idc parameter
         * 66  - Baseline profile
         * 77  - Main profile
         * 100 - High profile
         */
        mpp_enc_cfg_set_s32(cfg_, "h264:profile", 100);
        /*
         * H.264 level_idc parameter
         * 10 / 11 / 12 / 13    - qcif@15fps / cif@7.5fps / cif@15fps / cif@30fps
         * 20 / 21 / 22         - cif@30fps / half-D1@@25fps / D1@12.5fps
         * 30 / 31 / 32         - D1@25fps / 720p@30fps / 720p@60fps
         * 40 / 41 / 42         - 1080p@30fps / 1080p@30fps / 1080p@60fps
         * 50 / 51 / 52         - 4K@30fps
         */
        mpp_enc_cfg_set_s32(cfg_, "h264:level", 40);
        mpp_enc_cfg_set_s32(cfg_, "h264:cabac_en", 1);
        mpp_enc_cfg_set_s32(cfg_, "h264:cabac_idc", 0);
        mpp_enc_cfg_set_s32(cfg_, "h264:trans8x8", 1);

        // mpp_env_get_u32("constraint_set", &constraint_set, 0);
        // if (constraint_set & 0x3f0000)
        //     mpp_enc_cfg_set_s32(cfg_, "h264:constraint_set", constraint_set);
    } break;
    case MPP_VIDEO_CodingHEVC:
    case MPP_VIDEO_CodingMJPEG:
    case MPP_VIDEO_CodingVP8: {
    } break;
    default: {
        spdlog::error("unsupport encoder coding type {}", enc_info_.code_type);
    } break;
    }

    // p->split_mode = 0;
    // p->split_arg = 0;
    // p->split_out = 0;

    // mpp_env_get_u32("split_mode", &p->split_mode, MPP_ENC_SPLIT_NONE);
    // mpp_env_get_u32("split_arg", &p->split_arg, 0);
    // mpp_env_get_u32("split_out", &p->split_out, 0);

    // if (p->split_mode) {
    //     mpp_log_q(quiet, "%p split mode %d arg %d out %d\n", ctx,
    //         p->split_mode, p->split_arg, p->split_out);
    //     mpp_enc_cfg_set_s32(cfg_, "split:mode", p->split_mode);
    //     mpp_enc_cfg_set_s32(cfg_, "split:arg", p->split_arg);
    //     mpp_enc_cfg_set_s32(cfg_, "split:out", p->split_out);
    // }

    // mpp_env_get_u32("mirroring", &mirroring, 0);
    // mpp_env_get_u32("rotation", &rotation, 0);
    // mpp_env_get_u32("flip", &flip, 0);

    // mpp_enc_cfg_set_s32(cfg_, "prep:mirroring", mirroring);
    // mpp_enc_cfg_set_s32(cfg_, "prep:rotation", rotation);
    // mpp_enc_cfg_set_s32(cfg_, "prep:flip", flip);

    auto ret = api_->control(ctx_, MPP_ENC_SET_CFG, cfg_);
    if (ret) {
        spdlog::error("mpi control enc set cfg failed ret {}", magic_enum::enum_name(ret));
        return false;
    }
    return true;
}

bool VideoEncoder::Init(const std::function<void(uint8_t* data, uint32_t size)>& package_callback)
{
    MPP_RET ret = MPP_SUCCESS;
    enc_info_.width = frame_info_.width;
    enc_info_.height = frame_info_.height;
    enc_info_.code_type = AdaptStreamType(stream_info_.StreamType);
    enc_info_.frame_format = AdaptFrameType(frame_info_.format);
    enc_info_.hor_stride = MPP_ALIGN(frame_info_.width, 16);
    enc_info_.ver_stride = MPP_ALIGN(frame_info_.height, 16);
    enc_info_.frame_size = GetFrameSize(enc_info_.frame_format, enc_info_.hor_stride, enc_info_.ver_stride);
    enc_info_.header_size = GetHeaderSize(enc_info_.frame_format, frame_info_.width, frame_info_.height);
    enc_info_.mdinfo_size = (MPP_VIDEO_CodingHEVC == enc_info_.code_type) ? (MPP_ALIGN(enc_info_.hor_stride, 32) >> 5) * (MPP_ALIGN(enc_info_.ver_stride, 32) >> 5) * 16 : (MPP_ALIGN(enc_info_.hor_stride, 64) >> 6) * (MPP_ALIGN(enc_info_.ver_stride, 16) >> 4) * 16;
    enc_info_.bps = frame_info_.width * frame_info_.height / 8 * frame_info_.fps;
    enc_info_.rc_mode = MPP_ENC_RC_MODE_VBR;
    // auto ret = mpp_buffer_group_get_internal(&enc_info_.buf_grp, MPP_BUFFER_TYPE_DRM);
    // if (ret) {
    //     spdlog::error("failed to get mpp buffer group ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.frm_buf, enc_info_.frame_size + enc_info_.header_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for input frame ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.pkt_buf, enc_info_.frame_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for output packet ret {}", ret);
    //     return false;
    // }

    // ret = mpp_buffer_get(enc_info_.buf_grp, &enc_info_.md_info, enc_info_.mdinfo_size);
    // if (ret) {
    //     spdlog::error("failed to get buffer for motion info output packet ret {}", ret);
    //     return false;
    // }

    ret = mpp_check_support_format(MPP_CTX_ENC, enc_info_.code_type);
    if (ret == MPP_SUCCESS) {
        spdlog::info("Mpp Support {}", stream_info_.StreamType);
    } else {
        spdlog::error("Mpp Not Support {}", stream_info_.StreamType);
        return false;
    }

    ret = mpp_create(&ctx_, &api_);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Create mpp error {}", ret);
        return false;
    }

    spdlog::info("encoder start w {} h {} type {}",
        frame_info_.width, frame_info_.height, frame_info_.format);
    MppPollType timeout = MPP_POLL_NON_BLOCK;

    ret = api_->control(ctx_, MPP_SET_INPUT_TIMEOUT, &timeout);
    if (ret != MPP_SUCCESS) {
        spdlog::info("mpi control set input timeout {} ret {}", timeout, ret);
        return false;
    }
    timeout = MPP_POLL_BLOCK;
    ret = api_->control(ctx_, MPP_SET_OUTPUT_TIMEOUT, &timeout);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp Set output time out error {}", ret);
        return false;
    }

    ret = mpp_init(ctx_, MPP_CTX_ENC, enc_info_.code_type);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp Init error {}", ret);
        return false;
    }

    ret = mpp_enc_cfg_init(&cfg_);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Mpp enc_cfg_init error {}", ret);
        return false;
    }

    ret = api_->control(ctx_, MPP_ENC_GET_CFG, cfg_);
    if (ret) {
        spdlog::error("get enc cfg failed ret {}", ret);
        return false;
    }

    if(!SetMppEncCfg()) {
        return false;
    }

    package_callback_ = package_callback;
    recv_thread_ = std::thread(&VideoEncoder::EncRecvThread, this);

    return true;
}

#include <time.h>

static inline RK_S64 mpp_time()
{
    struct timespec time = {0, 0};
    clock_gettime(CLOCK_MONOTONIC, &time);
    return (RK_S64)time.tv_sec * 1000000 + (RK_S64)time.tv_nsec / 1000;
}

void VideoEncoder::EncRecvThread(VideoEncoder* self)
{
    MppPacket packet = NULL;
    RK_S64 last_pkt_time = 0;
    RK_S64 first_pkt_time = 0;
    RK_U32 eoi = 1;
    // std::shared_ptr<StreamPackage> stream_package;
    while (self->is_running_) {
        auto ret = self->api_->encode_get_packet(self->ctx_, &packet);
        if (ret || NULL == packet) {
            spdlog::error("Get Package error, {}", ret);
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
            continue;
        }
        last_pkt_time = mpp_time();

        auto data = (uint8_t*)mpp_packet_get_pos(packet);
        auto len = mpp_packet_get_length(packet);

        if(first_pkt_time == 0)
            first_pkt_time = mpp_time();

        auto pkt_eos = mpp_packet_get_eos(packet);
        /* for low delay partition encoding */
        if (mpp_packet_is_partition(packet)) {
            eoi = mpp_packet_is_eoi(packet);
        }

        self->package_callback_(data, len);

        ret = mpp_packet_deinit(&packet);
        assert(ret == MPP_SUCCESS);
    }
}

bool VideoEncoder::PutFrame(uint8_t* data, uint32_t size, int dma_fd)
{   
    MppBuffer cam_buf = NULL;
    MppFrame frame = nullptr;

    auto ret = mpp_frame_init(&frame);
    if (ret) {
        spdlog::error("mpp_frame_init failed {}", ret);
        return false;
    }

    mpp_frame_set_width(frame, enc_info_.width);
    mpp_frame_set_height(frame, enc_info_.height);
    mpp_frame_set_hor_stride(frame, enc_info_.hor_stride);
    mpp_frame_set_ver_stride(frame, enc_info_.ver_stride);
    mpp_frame_set_fmt(frame, enc_info_.frame_format);
    mpp_frame_set_eos(frame, 0);

    MppBufferInfo info;
    memset(&info, 0, sizeof(MppBufferInfo));
    info.type = MPP_BUFFER_TYPE_EXT_DMA;
    info.fd =  dma_fd;
    info.size = size & 0x07ffffff;
    info.index = (size & 0xf8000000) >> 27;
    mpp_buffer_import(&cam_buf, &info);

    mpp_frame_set_buffer(frame, cam_buf);

    ret = api_->encode_put_frame(ctx_, frame);
    if (ret != MPP_SUCCESS) {
        spdlog::error("Encode frame error {}", ret);
        return false;
    }

    return true;
}

liwang54321 commented 1 year ago

正常的画面应该是一个linux桌面

HermanChen commented 1 year ago

感觉stride配置有问题，RGB 的话，hor_stride 应该是 width 的 3 倍，RGBA 的话，是 4 倍

liwang54321 commented 1 year ago

hao

感觉stride配置有问题，RGB 的话，hor_stride 应该是 width 的 3 倍，RGBA 的话，是 4 倍感谢，我等下试一下，谢谢

haozizhuimao commented 10 months ago

bool VideoEncoder::PutFrame(uint8_t* data, uint32_t size, int dma_fd) @liwang54321 这个接口怎么使用下谢谢，而且我看第一个参数也没有使用，谢谢

liwang54321 commented 10 months ago

bool VideoEncoder::PutFrame(uint8_t* data, uint32_t size, int dma_fd) @liwang54321 这个接口怎么使用下谢谢，而且我看第一个参数也没有使用，谢谢我这个好像也有点忘了, 临时测试的. 应该是直接把图片数据和数据长度放进去就能用，最后一个参数应该可以空着

stz-source commented 4 months ago

请问你解决了吗，我也遇到了一样的问题

rockchip-linux / mpp

rk3588 视频编码，编码数据有问题 #343