shithub: openh264

Info  •  Files  •  Log  •  Branches

Merge pull request #1217 from ruil2/crash_update

fix crash bugs that too large size local varialbe will cause stack overflow

--- a/codec/encoder/core/inc/picture.h

+++ b/codec/encoder/core/inc/picture.h

@@ -54,6 +54,7 @@

 int32_t    iActualListSize;      // actual list size

 uint32_t uiSadCostThreshold[BLOCK_SIZE_ALL];

 bool      bRefBlockFeatureCalculated; // flag of whether pre-process is done

+uint16_t **pFeatureValuePointerList;//uint16_t* pFeatureValuePointerList[WELS_MAX (LIST_SIZE_SUM_16x16, LIST_SIZE_MSE_16x16)]

 } SScreenBlockFeatureStorage; //should be stored with RefPic, one for each frame

 /*

--- a/codec/encoder/core/src/svc_motion_estimate.cpp

+++ b/codec/encoder/core/src/svc_motion_estimate.cpp

@@ -622,6 +622,10 @@

   pScreenBlockFeatureStorage->pLocationPointer = (uint16_t*)pMa->WelsMalloc (2 * kiFrameSize * sizeof (uint16_t),

       "pScreenBlockFeatureStorage->pLocationPointer");

   WELS_VERIFY_RETURN_IF (ENC_RETURN_MEMALLOCERR, NULL == pScreenBlockFeatureStorage->pLocationPointer)

+    //  uint16_t* pFeatureValuePointerList[WELS_MAX (LIST_SIZE_SUM_16x16, LIST_SIZE_MSE_16x16)] = {0};

+  pScreenBlockFeatureStorage->pFeatureValuePointerList = (uint16_t**)pMa->WelsMalloc (WELS_MAX (LIST_SIZE_SUM_16x16, LIST_SIZE_MSE_16x16)* sizeof (uint16_t*),

+    "pScreenBlockFeatureStorage->pFeatureValuePointerList");

+  WELS_VERIFY_RETURN_IF (ENC_RETURN_MEMALLOCERR, NULL == pScreenBlockFeatureStorage->pFeatureValuePointerList)

   pScreenBlockFeatureStorage->pFeatureOfBlockPointer = NULL;

   pScreenBlockFeatureStorage->iIs16x16 = !bIsBlock8x8;

@@ -649,6 +653,11 @@

       pScreenBlockFeatureStorage->pLocationPointer = NULL;

     }

+    if (pScreenBlockFeatureStorage->pFeatureValuePointerList) {

+      pMa->WelsFree (pScreenBlockFeatureStorage->pFeatureValuePointerList, "pScreenBlockFeatureStorage->pFeatureValuePointerList");

+      pScreenBlockFeatureStorage->pFeatureValuePointerList = NULL;

+    }

+

     return ENC_RETURN_SUCCESS;

   }

   return ENC_RETURN_UNEXPECTED;

@@ -760,7 +769,6 @@

   const int32_t iWidth = pRef->iWidthInPixel - iEdgeDiscard;

   const int32_t kiHeight = pRef->iHeightInPixel - iEdgeDiscard;

   const int32_t kiActualListSize = pScreenBlockFeatureStorage->iActualListSize;

-  uint16_t* pFeatureValuePointerList[WELS_MAX (LIST_SIZE_SUM_16x16, LIST_SIZE_MSE_16x16)] = {0};

   memset (pTimesOfFeatureValue, 0, sizeof (int32_t)*kiActualListSize);

   (pFunc->pfCalculateBlockFeatureOfFrame[iIs16x16]) (pRefData, iWidth, kiHeight, iRefStride, pFeatureOfBlock,

@@ -768,10 +776,10 @@

   //assign pLocationOfFeature pointer

   InitializeHashforFeature_c (pTimesOfFeatureValue, pBuf, kiActualListSize,

-                              pLocationOfFeature, pFeatureValuePointerList);

+                              pLocationOfFeature, pScreenBlockFeatureStorage->pFeatureValuePointerList);

   //assign each pixel's pLocationOfFeature

-  FillQpelLocationByFeatureValue_c (pFeatureOfBlock, iWidth, kiHeight, pFeatureValuePointerList);

+  FillQpelLocationByFeatureValue_c (pFeatureOfBlock, iWidth, kiHeight, pScreenBlockFeatureStorage->pFeatureValuePointerList);

   return true;

 }