shithub: openh264

Download patch

ref: 914302a46263d6042d70853a47f5d636ac5ea7a5
parent: aaa25160ec86fcc1b8fad5c172a77b1a4fe6e710
author: sijchen <[email protected]>
date: Wed Feb 10 05:18:52 EST 2016 

avoid memory problem if mem alloc failed in the middle of InitDqLayer


--- a/codec/encoder/core/inc/svc_enc_frame.h
+++ b/codec/encoder/core/inc/svc_enc_frame.h
@@ -105,6 +105,7 @@
 SPicture*               pDecPic;        // reconstruction picture pointer for layer
 SPicture*               pRefOri[MAX_REF_PIC_COUNT];
 
+int32_t                 iMaxSliceNum;
 int32_t*                pNumSliceCodedOfPartition;      // for dynamic slicing mode
 int32_t*                pLastCodedMbIdxOfPartition;     // for dynamic slicing mode
 int32_t*                pLastMbIdxOfPartition;          // for dynamic slicing mode
--- a/codec/encoder/core/src/encoder_ext.cpp
+++ b/codec/encoder/core/src/encoder_ext.cpp
@@ -952,6 +952,53 @@
   }
 }
 
+void FreeDqLayer (SDqLayer* pDq, CMemoryAlign* pMa) {
+  if (NULL == pDq) {
+    return;
+  }
+
+  if (NULL != pDq->sLayerInfo.pSliceInLayer) {
+    int32_t iSliceIdx = 0;
+    while (iSliceIdx < pDq->iMaxSliceNum) {
+      SSlice* pSlice = &pDq->sLayerInfo.pSliceInLayer[iSliceIdx];
+      FreeMbCache (&pSlice->sMbCacheInfo, pMa);
+
+      //slice bs buffer
+      if (NULL != pSlice->sSliceBs.pBs) {
+        pMa->WelsFree (pSlice->sSliceBs.pBs, "sSliceBs.pBs");
+        pSlice->sSliceBs.pBs = NULL;
+      }
+      ++ iSliceIdx;
+    }
+    pMa->WelsFree (pDq->sLayerInfo.pSliceInLayer, "pSliceInLayer");
+    pDq->sLayerInfo.pSliceInLayer = NULL;
+  }
+
+  if (pDq->pNumSliceCodedOfPartition) {
+    pMa->WelsFree (pDq->pNumSliceCodedOfPartition, "pNumSliceCodedOfPartition");
+    pDq->pNumSliceCodedOfPartition = NULL;
+  }
+
+  if (pDq->pLastCodedMbIdxOfPartition) {
+    pMa->WelsFree (pDq->pLastCodedMbIdxOfPartition, "pLastCodedMbIdxOfPartition");
+    pDq->pLastCodedMbIdxOfPartition = NULL;
+  }
+  if (pDq->pLastMbIdxOfPartition) {
+    pMa->WelsFree (pDq->pLastMbIdxOfPartition, "pLastMbIdxOfPartition");
+    pDq->pLastMbIdxOfPartition = NULL;
+  }
+
+  if (pDq->pFeatureSearchPreparation) {
+    ReleaseFeatureSearchPreparation (pMa, pDq->pFeatureSearchPreparation->pFeatureOfBlock);
+    pMa->WelsFree (pDq->pFeatureSearchPreparation, "pFeatureSearchPreparation");
+    pDq->pFeatureSearchPreparation = NULL;
+  }
+
+  UninitSlicePEncCtx (pDq, pMa);
+  pDq->iMaxSliceNum = 0;
+}
+
+
 static int32_t WelsGenerateNewSps (sWelsEncCtx* pCtx, const bool kbUseSubsetSps, const int32_t iDlayerIndex,
                                    const int32_t iDlayerCount, const int32_t kiSpsId,
                                    SWelsSPS*& pSps, SSubsetSps*& pSubsetSps, bool bSVCBaselayer) {
@@ -1209,10 +1256,6 @@
     SSpatialLayerInternal* pParamInternal    = &pParam->sDependencyLayers[iDlayerIndex];
     const int32_t kiMbW             = (pDlayer->iVideoWidth + 0x0f) >> 4;
     const int32_t kiMbH             = (pDlayer->iVideoHeight + 0x0f) >> 4;
-    int32_t iMaxSliceNum            = 1;
-    const int32_t kiSliceNum = GetInitialSliceNum (kiMbW, kiMbH, &pDlayer->sSliceArgument);
-    if (iMaxSliceNum < kiSliceNum)
-      iMaxSliceNum = kiSliceNum;
 
     pParamInternal->iSkipFrameFlag = 0;
     pParamInternal->iCodingIndex = 0;
@@ -1222,8 +1265,14 @@
     pParamInternal->bEncCurFrmAsIdrFlag = true;  // make sure first frame is IDR
     // pDq layers list
     pDqLayer = (SDqLayer*)pMa->WelsMallocz (sizeof (SDqLayer), "pDqLayer");
-    WELS_VERIFY_RETURN_PROC_IF (1, (NULL == pDqLayer), FreeMemorySvc (ppCtx))
+    WELS_VERIFY_RETURN_PROC_IF (1, (NULL == pDqLayer), FreeDqLayer (pDqLayer, pMa))
 
+    int32_t iMaxSliceNum            = 1;
+    const int32_t kiSliceNum = GetInitialSliceNum (kiMbW, kiMbH, &pDlayer->sSliceArgument);
+    if (iMaxSliceNum < kiSliceNum)
+      iMaxSliceNum = kiSliceNum;
+    pDqLayer->iMaxSliceNum = iMaxSliceNum;
+
     // for dynamic slicing mode
     if (SM_SIZELIMITED_SLICE == pDlayer->sSliceArgument.uiSliceMode) {
       const int32_t iSize                       = pParam->iMultipleThreadIdc * sizeof (int32_t);
@@ -1231,12 +1280,11 @@
       pDqLayer->pNumSliceCodedOfPartition       = (int32_t*)pMa->WelsMallocz (iSize, "pNumSliceCodedOfPartition");
       pDqLayer->pLastCodedMbIdxOfPartition      = (int32_t*)pMa->WelsMallocz (iSize, "pLastCodedMbIdxOfPartition");
       pDqLayer->pLastMbIdxOfPartition           = (int32_t*)pMa->WelsMallocz (iSize, "pLastMbIdxOfPartition");
-
       WELS_VERIFY_RETURN_PROC_IF (1,
                                   (NULL == pDqLayer->pNumSliceCodedOfPartition ||
                                    NULL == pDqLayer->pLastCodedMbIdxOfPartition ||
                                    NULL == pDqLayer->pLastMbIdxOfPartition),
-                                  FreeMemorySvc (ppCtx))
+                                  FreeDqLayer (pDqLayer, pMa))
     }
     pDqLayer->bNeedAdjustingSlicing = false;
 
@@ -1244,10 +1292,10 @@
     pDqLayer->iMbHeight = kiMbH;
     {
       pDqLayer->sLayerInfo.pSliceInLayer = (SSlice*)pMa->WelsMallocz (sizeof (SSlice) * iMaxSliceNum, "pSliceInLayer");
-      WELS_VERIFY_RETURN_PROC_IF (1, (NULL == pDqLayer->sLayerInfo.pSliceInLayer), FreeMemorySvc (ppCtx))
+      WELS_VERIFY_RETURN_PROC_IF (1, (NULL == pDqLayer->sLayerInfo.pSliceInLayer), FreeDqLayer (pDqLayer, pMa))
 
       int32_t iReturn = InitpSliceInLayer (ppCtx, pDqLayer, pMa, iMaxSliceNum, iDlayerIndex);
-      WELS_VERIFY_RETURN_PROC_IF (1, (ENC_RETURN_SUCCESS != iReturn), FreeMemorySvc (ppCtx))
+      WELS_VERIFY_RETURN_PROC_IF (1, (ENC_RETURN_SUCCESS != iReturn), FreeDqLayer (pDqLayer, pMa))
     }
 
     //deblocking parameters initialization
@@ -1916,12 +1964,6 @@
     ResetLtrState (& (*ppCtx)->pLtr[i]);
   }
 
-  (*ppCtx)->ppRefPicListExt = (SRefList**)pMa->WelsMalloc (kiNumDependencyLayers * sizeof (SRefList*), "ppRefPicListExt");
-  WELS_VERIFY_RETURN_PROC_IF (1, (NULL == (*ppCtx)->ppRefPicListExt), FreeMemorySvc (ppCtx))
-
-  (*ppCtx)->ppDqLayerList = (SDqLayer**)pMa->WelsMalloc (kiNumDependencyLayers * sizeof (SDqLayer*), "ppDqLayerList");
-  WELS_VERIFY_RETURN_PROC_IF (1, (NULL == (*ppCtx)->ppDqLayerList), FreeMemorySvc (ppCtx))
-
   // stride tables
   if (AllocStrideTables (ppCtx, kiNumDependencyLayers)) {
     WelsLog (& (*ppCtx)->sLogCtx, WELS_LOG_WARNING, "RequestMemorySvc(), AllocStrideTables failed!");
@@ -1987,6 +2029,17 @@
 
   //End of pVaa memory allocation
 
+  (*ppCtx)->ppRefPicListExt = (SRefList**)pMa->WelsMalloc (kiNumDependencyLayers * sizeof (SRefList*), "ppRefPicListExt");
+  WELS_VERIFY_RETURN_PROC_IF (1, (NULL == (*ppCtx)->ppRefPicListExt), FreeMemorySvc (ppCtx))
+
+  (*ppCtx)->ppDqLayerList = (SDqLayer**)pMa->WelsMalloc (kiNumDependencyLayers * sizeof (SDqLayer*), "ppDqLayerList");
+  WELS_VERIFY_RETURN_PROC_IF (1, (NULL == (*ppCtx)->ppDqLayerList), FreeMemorySvc (ppCtx))
+
+  for (int32_t k = 0; k < kiNumDependencyLayers; k++) {
+    (*ppCtx)->ppRefPicListExt[k] = NULL;
+    (*ppCtx)->ppDqLayerList[k] = NULL;
+  }
+
   iResult = InitDqLayers (ppCtx, pExistingParasetList);
   if (iResult) {
     WelsLog (& (*ppCtx)->sLogCtx, WELS_LOG_WARNING, "RequestMemorySvc(), InitDqLayers failed(%d)!", iResult);
@@ -2143,53 +2196,9 @@
         SDqLayer* pDq = pCtx->ppDqLayerList[ilayer];
         SSpatialLayerConfig* pDlp = &pCtx->pSvcParam->sSpatialLayers[ilayer];
 
-        const bool kbIsDynamicSlicing = (SM_SIZELIMITED_SLICE == pDlp->sSliceArgument.uiSliceMode);
-
         // pDq layers
         if (NULL != pDq) {
-          if (NULL != pDq->sLayerInfo.pSliceInLayer) {
-            int32_t iSliceIdx = 0;
-            int32_t iSliceNum = GetInitialSliceNum (pDq->iMbWidth, pDq->iMbHeight, &pDlp->sSliceArgument);
-            if (pDlp->sSliceArgument.uiSliceMode == SM_SIZELIMITED_SLICE && pCtx->iActiveThreadsNum == 1) {
-              if (iSliceNum < pDq->sSliceEncCtx.iMaxSliceNumConstraint) {
-                iSliceNum = pDq->sSliceEncCtx.iMaxSliceNumConstraint;
-              }
-            }
-            if (iSliceNum < 1)
-              iSliceNum = 1;
-            while (iSliceIdx < iSliceNum) {
-              SSlice* pSlice = &pDq->sLayerInfo.pSliceInLayer[iSliceIdx];
-              FreeMbCache (&pSlice->sMbCacheInfo, pMa);
-
-              //slice bs buffer
-              if (NULL != pSlice->sSliceBs.pBs) {
-                pMa->WelsFree (pSlice->sSliceBs.pBs, "sSliceBs.pBs");
-                pSlice->sSliceBs.pBs = NULL;
-              }
-              ++ iSliceIdx;
-            }
-            pMa->WelsFree (pDq->sLayerInfo.pSliceInLayer, "pSliceInLayer");
-            pDq->sLayerInfo.pSliceInLayer = NULL;
-          }
-          if (kbIsDynamicSlicing) {
-            pMa->WelsFree (pDq->pNumSliceCodedOfPartition, "pNumSliceCodedOfPartition");
-            pDq->pNumSliceCodedOfPartition = NULL;
-            pMa->WelsFree (pDq->pLastCodedMbIdxOfPartition, "pLastCodedMbIdxOfPartition");
-            pDq->pLastCodedMbIdxOfPartition = NULL;
-            pMa->WelsFree (pDq->pLastMbIdxOfPartition, "pLastMbIdxOfPartition");
-            pDq->pLastMbIdxOfPartition = NULL;
-          }
-
-          if (pDq->pFeatureSearchPreparation) {
-            ReleaseFeatureSearchPreparation (pMa, pDq->pFeatureSearchPreparation->pFeatureOfBlock);
-            pMa->WelsFree (pDq->pFeatureSearchPreparation, "pFeatureSearchPreparation");
-            pDq->pFeatureSearchPreparation = NULL;
-          }
-
-          if (NULL != pDq) {
-            UninitSlicePEncCtx (pDq, pMa);
-          }
-
+          FreeDqLayer (pDq, pMa);
           pMa->WelsFree (pDq, "pDq");
           pDq = NULL;
           pCtx->ppDqLayerList[ilayer] = NULL;
--- a/codec/encoder/core/src/svc_enc_slice_segment.cpp
+++ b/codec/encoder/core/src/svc_enc_slice_segment.cpp
@@ -469,11 +469,13 @@
       pSliceSeg->pOverallMbMap = NULL;
     }
 
-    pSliceSeg->iMbNumInFrame    = 0;
+    pSliceSeg->uiSliceMode      = SM_SINGLE_SLICE;      // single in default
     pSliceSeg->iMbWidth         = 0;
     pSliceSeg->iMbHeight        = 0;
-    pSliceSeg->uiSliceMode      = SM_SINGLE_SLICE;      // single in default
     pSliceSeg->iSliceNumInFrame = 0;
+    pSliceSeg->iMbNumInFrame    = 0;
+    pSliceSeg->uiSliceSizeConstraint = 0;
+    pSliceSeg->iMaxSliceNumConstraint    = 0;
   }
 }