search

Xilinx / vcu-ctrl-sw

Other
38 stars 30 forks source link

ctrlsw_decoder mistakenly decode nv16 as xv20 for avc because lib_decode get wrong bitdepth #5

Closed njliang closed 3 years ago

njliang commented 3 years ago

getMaxBitDepthFromProfile return 10 for avc nv16 (profile: AL_PROFILE_AVC_HIGH_422)

static int getMaxBitDepthFromProfile(int profile_idc)
{
  if(
    (profile_idc == AL_GET_PROFILE_IDC(AL_PROFILE_AVC_BASELINE))
    || (profile_idc == AL_GET_PROFILE_IDC(AL_PROFILE_AVC_MAIN))
    || (profile_idc == AL_GET_PROFILE_IDC(AL_PROFILE_AVC_EXTENDED))
    || (profile_idc == AL_GET_PROFILE_IDC(AL_PROFILE_AVC_HIGH))
    )
    return 8;
  return 10;
}

static int getMaxBitDepth(AL_TAvcSps const* pSPS)
{
  int iSPSLumaBitDepth = pSPS->bit_depth_luma_minus8 + 8;
  int iSPSChromaBitDepth = pSPS->bit_depth_chroma_minus8 + 8;
  int iMaxSPSBitDepth = Max(iSPSLumaBitDepth, iSPSChromaBitDepth);
  int iMaxBitDepth = Max(iMaxSPSBitDepth, getMaxBitDepthFromProfile(pSPS->profile_idc));

  if((iMaxBitDepth % 2) != 0)
    iMaxBitDepth++;
  return iMaxBitDepth;
}