ref: fc56c7d007762e4dc95e04517cc19326329e1eae
parent: 99412b0fc228beb56efaaaad25e3b6b543dbb519
author: Licai Guo <[email protected]>
date: Wed Jan 22 19:07:21 EST 2014
1. use WELS_CLIP1 to replace table look-up; 2. fix bugs about buffer overflow 3. add more boundary checks
--- a/codec/console/dec/src/h264dec.cpp
+++ b/codec/console/dec/src/h264dec.cpp
@@ -81,7 +81,7 @@
int32_t iColorFormat = videoFormatInternal;
static int32_t iFrameNum = 0;
- EDecodeMode eDecoderMode = AUTO_MODE;
+ EDecodeMode eDecoderMode = SW_MODE;
EBufferProperty eOutputProperty = BUFFER_DEVICE;
CUtils cOutputModule;
--- a/codec/decoder/core/inc/error_code.h
+++ b/codec/decoder/core/inc/error_code.h
@@ -57,7 +57,7 @@
*
*/
#define GENERATE_ERROR_NO(iErrLevel, iErrInfo) ((iErrLevel << 16) | (iErrInfo & 0xFFFF))
-
+#define ERR_INVALID_INTRA4X4_MODE -1
/* ERR_LEVEL */
//-----------------------------------------------------------------------------------------------------------
--- a/codec/decoder/core/inc/parse_mb_syn_cavlc.h
+++ b/codec/decoder/core/inc/parse_mb_syn_cavlc.h
@@ -56,9 +56,9 @@
#define CHROMA_AC 5
typedef struct TagReadBitsCache {-uint32_t uiCache32Bit;
-uint8_t uiRemainBits;
-uint8_t* pBuf;
+ uint32_t uiCache32Bit;
+ uint8_t uiRemainBits;
+ uint8_t* pBuf;
} SReadBitsCache;
#define SHIFT_BUFFER(pBitsCache) { pBitsCache->pBuf+=2; pBitsCache->uiRemainBits += 16; pBitsCache->uiCache32Bit |= (((pBitsCache->pBuf[2] << 8) | pBitsCache->pBuf[3]) << (32 - pBitsCache->uiRemainBits)); }@@ -65,51 +65,51 @@
#define POP_BUFFER(pBitsCache, iCount) { pBitsCache->uiCache32Bit <<= iCount; pBitsCache->uiRemainBits -= iCount; } static const uint8_t g_kuiZigzagScan[16] = { //4*4block residual zig-zag scan order-0, 1, 4, 8,
-5, 2, 3, 6,
-9, 12, 13, 10,
-7, 11, 14, 15,
+ 0, 1, 4, 8,
+ 5, 2, 3, 6,
+ 9, 12, 13, 10,
+ 7, 11, 14, 15,
};
typedef struct TagI16PredInfo {-int8_t iPredMode;
-int8_t iLeftAvail;
-int8_t iTopAvail;
-int8_t iLeftTopAvail;
+ int8_t iPredMode;
+ int8_t iLeftAvail;
+ int8_t iTopAvail;
+ int8_t iLeftTopAvail;
} SI16PredInfo;
static const SI16PredInfo g_ksI16PredInfo[4] = {-{I16_PRED_V, 0, 1, 0},-{I16_PRED_H, 1, 0, 0},-{ 0, 0, 0, 0},-{I16_PRED_P, 1, 1, 1},+ {I16_PRED_V, 0, 1, 0},+ {I16_PRED_H, 1, 0, 0},+ { 0, 0, 0, 0},+ {I16_PRED_P, 1, 1, 1},};
static const SI16PredInfo g_ksChromaPredInfo[4] = {-{ 0, 0, 0, 0},-{C_PRED_H, 1, 0, 0},-{C_PRED_V, 0, 1, 0},-{C_PRED_P, 1, 1, 1},+ { 0, 0, 0, 0},+ {C_PRED_H, 1, 0, 0},+ {C_PRED_V, 0, 1, 0},+ {C_PRED_P, 1, 1, 1},};
typedef struct TagI4PredInfo {-int8_t iPredMode;
-int8_t iLeftAvail;
-int8_t iTopAvail;
-int8_t iLeftTopAvail;
+ int8_t iPredMode;
+ int8_t iLeftAvail;
+ int8_t iTopAvail;
+ int8_t iLeftTopAvail;
// int8_t right_top_avail; //when right_top unavailable but top avail, we can pad the right-top with the rightmost pixel of top
} SI4PredInfo;
static const SI4PredInfo g_ksI4PredInfo[9] = {-{ I4_PRED_V, 0, 1, 0},-{ I4_PRED_H, 1, 0, 0},-{ 0, 0, 0, 0},-{I4_PRED_DDL, 0, 1, 0},-{I4_PRED_DDR, 1, 1, 1},-{ I4_PRED_VR, 1, 1, 1},-{ I4_PRED_HD, 1, 1, 1},-{ I4_PRED_VL, 0, 1, 0},-{ I4_PRED_HU, 1, 0, 0},+ { I4_PRED_V, 0, 1, 0},+ { I4_PRED_H, 1, 0, 0},+ { 0, 0, 0, 0},+ {I4_PRED_DDL, 0, 1, 0},+ {I4_PRED_DDR, 1, 1, 1},+ { I4_PRED_VR, 1, 1, 1},+ { I4_PRED_HD, 1, 1, 1},+ { I4_PRED_VL, 0, 1, 0},+ { I4_PRED_HU, 1, 0, 0},};
static const uint8_t g_kuiI16CbpTable[6] = {0, 16, 32, 15, 31, 47}; //reference to JM@@ -116,22 +116,22 @@
typedef struct TagPartMbInfo {-MbType iType;
-int8_t iPartCount; //P_16*16, P_16*8, P_8*16, P_8*8 based on 8*8 block; P_8*4, P_4*8, P_4*4 based on 4*4 block
-int8_t iPartWidth; //based on 4*4 block
+ MbType iType;
+ int8_t iPartCount; //P_16*16, P_16*8, P_8*16, P_8*8 based on 8*8 block; P_8*4, P_4*8, P_4*4 based on 4*4 block
+ int8_t iPartWidth; //based on 4*4 block
} SPartMbInfo;
static const SPartMbInfo g_ksInterMbTypeInfo[5] = {-{MB_TYPE_16x16, 1, 4},-{MB_TYPE_16x8, 2, 4},-{MB_TYPE_8x16, 2, 2},-{MB_TYPE_8x8, 4, 4},-{MB_TYPE_8x8_REF0, 4, 4}, //ref0--ref_idx not present in bit-stream and default as 0+ {MB_TYPE_16x16, 1, 4},+ {MB_TYPE_16x8, 2, 4},+ {MB_TYPE_8x16, 2, 2},+ {MB_TYPE_8x8, 4, 4},+ {MB_TYPE_8x8_REF0, 4, 4}, //ref0--ref_idx not present in bit-stream and default as 0};
static const SPartMbInfo g_ksInterSubMbTypeInfo[4] = {-{SUB_MB_TYPE_8x8, 1, 2},-{SUB_MB_TYPE_8x4, 2, 2},-{SUB_MB_TYPE_4x8, 2, 1},-{SUB_MB_TYPE_4x4, 4, 1},+ {SUB_MB_TYPE_8x8, 1, 2},+ {SUB_MB_TYPE_8x4, 2, 2},+ {SUB_MB_TYPE_4x8, 2, 1},+ {SUB_MB_TYPE_4x4, 4, 1},};
void_t GetNeighborAvailMbType (PNeighAvail pNeighAvail, PDqLayer pCurLayer);
--- a/codec/decoder/core/src/au_parser.cpp
+++ b/codec/decoder/core/src/au_parser.cpp
@@ -220,6 +220,21 @@
}
DecodeNalHeaderExt (pCurNal, pNal);
+ if ((pCurNal->sNalHeaderExt.uiQualityId != 0) || (pCurNal->sNalHeaderExt.bUseRefBasePicFlag != 0)) {+ WelsLog (pCtx, WELS_LOG_WARNING,
+ "ParseNalHeader() in Prefix Nal Unit:uiQualityId (%d) != 0, bUseRefBasePicFlag (%d) != 0, not supported!\n",
+ pCurNal->sNalHeaderExt.uiQualityId, pCurNal->sNalHeaderExt.bUseRefBasePicFlag);
+ PAccessUnit pCurAu = pCtx->pAccessUnitList;
+ uint32_t uiAvailNalNum = pCurAu->uiAvailUnitsNum;
+ ForceClearCurrentNal (pCurAu);
+
+ if (uiAvailNalNum > 1) {+ pCurAu->uiEndPos = uiAvailNalNum - 2;
+ pCtx->bAuReadyFlag = true;
+ }
+ pCtx->iErrorCode |= dsInvalidArgument;
+ return NULL;
+ }
pNal += NAL_UNIT_HEADER_EXT_SIZE;
iNalSize -= NAL_UNIT_HEADER_EXT_SIZE;
--- a/codec/decoder/core/src/decode_mb_aux.cpp
+++ b/codec/decoder/core/src/decode_mb_aux.cpp
@@ -93,13 +93,13 @@
int32_t kT3 = (32 + kT1 + kT2) >> 6;
int32_t kT4 = (32 + kT1 - kT2) >> 6;
- pDst[i] = pClip[ kT3 + pPred[i] ];
- pDst[i + kiStride3] = pClip[ kT4 + pPred[i + kiStride3] ];
+ pDst[i] = WELS_CLIP1( kT3 + pPred[i] );
+ pDst[i + kiStride3] = WELS_CLIP1( kT4 + pPred[i + kiStride3] );
kT1 = iSrc[i] - iSrc[i + 8];
kT2 = (iSrc[i + 4] >> 1) - iSrc[i + 12];
- pDst[i + kiStride] = pClip[ ((32 + kT1 + kT2) >> 6) + pDst[i + kiStride] ];
- pDst[i + kiStride2] = pClip[ ((32 + kT1 - kT2) >> 6) + pDst[i + kiStride2] ];
+ pDst[i + kiStride] = WELS_CLIP1( ((32 + kT1 + kT2) >> 6) + pDst[i + kiStride] );
+ pDst[i + kiStride2] = WELS_CLIP1( ((32 + kT1 - kT2) >> 6) + pDst[i + kiStride2] );
}
}
@@ -124,4 +124,4 @@
}
}
-} // namespace WelsDec
\ No newline at end of file
+} // namespace WelsDec
--- a/codec/decoder/core/src/decoder_core.cpp
+++ b/codec/decoder/core/src/decoder_core.cpp
@@ -519,6 +519,11 @@
if (uiSliceType > 4)
uiSliceType -= 5;
+ if((eNalType == NAL_UNIT_CODED_SLICE_IDR) && (uiSliceType != 2)){+ WelsLog (pCtx, WELS_LOG_WARNING, "Invalid slice type(%d) in IDR picture. \n", uiSliceType);
+ return GENERATE_ERROR_NO (ERR_LEVEL_SLICE_HEADER, ERR_INFO_INVALID_SLICE_TYPE);
+ }
+
if (kbExtensionFlag) { if (uiSliceType > 2) {WelsLog (pCtx, WELS_LOG_WARNING, "Invalid slice type(%d).\n", uiSliceType);
--- a/codec/decoder/core/src/parse_mb_syn_cavlc.cpp
+++ b/codec/decoder/core/src/parse_mb_syn_cavlc.cpp
@@ -390,7 +390,7 @@
int32_t bLeftTopAvail = uiSampleAvail & 0x02;
int32_t iTopAvail = uiSampleAvail & 0x01;
- if (*pMode > MAX_PRED_MODE_ID_I16x16) {+ if ((*pMode < 0) || (*pMode > MAX_PRED_MODE_ID_I16x16)) {return ERR_INFO_INVALID_I16x16_PRED_MODE;
}
@@ -419,10 +419,6 @@
int32_t bLeftTopAvail = uiSampleAvail & 0x02;
int32_t iTopAvail = uiSampleAvail & 0x01;
- if (*pMode > MAX_PRED_MODE_ID_CHROMA) {- return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
- }
-
if (C_PRED_DC == *pMode) { if (iLeftAvail && iTopAvail) {return 0;
@@ -451,8 +447,8 @@
int8_t iFinalMode;
- if (*pMode > MAX_PRED_MODE_ID_I4x4) {- return -1;
+ if ((*pMode < 0) || (*pMode > MAX_PRED_MODE_ID_I4x4)) {+ return ERR_INVALID_INTRA4X4_MODE;
}
if (I4_PRED_DC == *pMode) {@@ -468,7 +464,7 @@
} else {bool_t bModeAvail = CHECK_I4_MODE (*pMode, iLeftAvail, iTopAvail, bLeftTopAvail);
if (0 == bModeAvail) {- return -1;
+ return ERR_INVALID_INTRA4X4_MODE;
}
iFinalMode = *pMode;
@@ -746,7 +742,7 @@
iZerosLeft = 0;
}
- if (iZerosLeft < 0) {+ if ((iZerosLeft < 0) || ((iZerosLeft + uiTotalCoeff) > iMaxNumCoeff)) {return ERR_INFO_CAVLC_INVALID_ZERO_LEFT;
}
if ((i = CavlcGetRunBefore (iRun, &sReadBitsCache, uiTotalCoeff, pVlcTable, iZerosLeft)) == -1) {@@ -803,7 +799,7 @@
int32_t iFinalMode, i;
uint8_t uiNeighAvail = 0;
-
+ uint32_t uiTmp;
if (pNeighAvail->iLeftAvail) { //leftiSampleAvail[ 6] =
iSampleAvail[12] =
@@ -842,7 +838,7 @@
}
iFinalMode = CheckIntra4x4PredMode (&iSampleAvail[0], &iBestMode, i);
- if (iFinalMode < 0) {+ if (iFinalMode == ERR_INVALID_INTRA4X4_MODE) {return ERR_INFO_INVALID_I4x4_PRED_MODE;
}
@@ -856,11 +852,14 @@
pCurDqLayer->pIntraPredMode[iMbXy][4] = pIntraPredMode[4 + 8 * 1];
pCurDqLayer->pIntraPredMode[iMbXy][5] = pIntraPredMode[4 + 8 * 2];
pCurDqLayer->pIntraPredMode[iMbXy][6] = pIntraPredMode[4 + 8 * 3];
- pCurDqLayer->pChromaPredMode[iMbXy] = BsGetUe (pBs);
- if (-1 == pCurDqLayer->pChromaPredMode[iMbXy]
- || CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ uiTmp = BsGetUe (pBs);
+ if (uiTmp > MAX_PRED_MODE_ID_CHROMA) {return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
}
+ pCurDqLayer->pChromaPredMode[iMbXy] = uiTmp;
+ if (CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
+ }
return 0;
}
@@ -872,7 +871,7 @@
int32_t iFinalMode, i;
uint8_t uiNeighAvail = 0;
-
+ uint32_t uiTmp;
if (pNeighAvail->iLeftAvail && IS_INTRA (pNeighAvail->iLeftType)) { //leftiSampleAvail[ 6] =
iSampleAvail[12] =
@@ -911,7 +910,7 @@
}
iFinalMode = CheckIntra4x4PredMode (&iSampleAvail[0], &iBestMode, i);
- if (iFinalMode < 0) {+ if (iFinalMode == ERR_INVALID_INTRA4X4_MODE) {return ERR_INFO_INVALID_I4x4_PRED_MODE;
}
@@ -925,12 +924,14 @@
pCurDqLayer->pIntraPredMode[iMbXy][4] = pIntraPredMode[4 + 8 * 1];
pCurDqLayer->pIntraPredMode[iMbXy][5] = pIntraPredMode[4 + 8 * 2];
pCurDqLayer->pIntraPredMode[iMbXy][6] = pIntraPredMode[4 + 8 * 3];
-
- pCurDqLayer->pChromaPredMode[iMbXy] = BsGetUe (pBs);
- if (-1 == pCurDqLayer->pChromaPredMode[iMbXy]
- || CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ uiTmp = BsGetUe (pBs);
+ if (uiTmp > MAX_PRED_MODE_ID_CHROMA) {return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
}
+ pCurDqLayer->pChromaPredMode[iMbXy] = uiTmp;
+ if (CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
+ }
return 0;
}
@@ -938,7 +939,7 @@
int32_t ParseIntra16x16ModeConstrain0 (PNeighAvail pNeighAvail, PBitStringAux pBs, PDqLayer pCurDqLayer) {int32_t iMbXy = pCurDqLayer->iMbXyIndex;
uint8_t uiNeighAvail = 0; //0x07 = 0 1 1 1, means left, top-left, top avail or not. (1: avail, 0: unavail)
-
+ uint32_t uiTmp;
if (pNeighAvail->iLeftAvail) {uiNeighAvail = (1 << 2);
}
@@ -953,10 +954,13 @@
&pCurDqLayer->pIntraPredMode[iMbXy][7])) { //invalid iPredMode, must stop decodingreturn ERR_INFO_INVALID_I16x16_PRED_MODE;
}
- pCurDqLayer->pChromaPredMode[iMbXy] = BsGetUe (pBs);
+ uiTmp = BsGetUe (pBs);
+ if (uiTmp > MAX_PRED_MODE_ID_CHROMA) {+ return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
+ }
+ pCurDqLayer->pChromaPredMode[iMbXy] = uiTmp;
- if (-1 == pCurDqLayer->pChromaPredMode[iMbXy]
- || CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ if (CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
}
@@ -966,7 +970,7 @@
int32_t ParseIntra16x16ModeConstrain1 (PNeighAvail pNeighAvail, PBitStringAux pBs, PDqLayer pCurDqLayer) {int32_t iMbXy = pCurDqLayer->iMbXyIndex;
uint8_t uiNeighAvail = 0; //0x07 = 0 1 1 1, means left, top-left, top avail or not. (1: avail, 0: unavail)
-
+ uint32_t uiTmp;
if (pNeighAvail->iLeftAvail && IS_INTRA (pNeighAvail->iLeftType)) {uiNeighAvail = (1 << 2);
}
@@ -981,10 +985,13 @@
&pCurDqLayer->pIntraPredMode[iMbXy][7])) { //invalid iPredMode, must stop decodingreturn ERR_INFO_INVALID_I16x16_PRED_MODE;
}
- pCurDqLayer->pChromaPredMode[iMbXy] = BsGetUe (pBs);
+ uiTmp = BsGetUe (pBs);
+ if (uiTmp > MAX_PRED_MODE_ID_CHROMA) {+ return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
+ }
+ pCurDqLayer->pChromaPredMode[iMbXy] = uiTmp;
- if (-1 == pCurDqLayer->pChromaPredMode[iMbXy]
- || CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {+ if (CheckIntraChromaPredMode (uiNeighAvail, &pCurDqLayer->pChromaPredMode[iMbXy])) {return ERR_INFO_INVALID_I_CHROMA_PRED_MODE;
}