shithub: openh264

Download patch

ref: 5b004048ed394bc0318f8cfa7ab4a9055aa3f8eb
parent: 4486e328d4a480dfa386729b6c0157ad15d3dcf6
parent: 32ce337eb09a64bc4d97803c08cc0aefd757f840
author: huili2 <[email protected]>
date: Tue Jul 1 11:25:59 EDT 2014 

Merge pull request #1051 from ruil2/overflow_check

check each WriteBlockResidualCavlc to avoid overflow


--- a/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp
+++ b/codec/encoder/core/src/svc_set_mb_syn_cavlc.cpp
@@ -282,7 +282,8 @@
         iA = pNonZeroCoeffCount[iIdx - 1];
         iB = pNonZeroCoeffCount[iIdx - 8];
         WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-        WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, LUMA_AC, iC, pBs);
+        if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, LUMA_AC, iC, pBs))
+          return ENC_RETURN_VLCOVERFLOWFOUND;
         pBlock += 16;
       }
     }
@@ -301,22 +302,26 @@
           iA = pNonZeroCoeffCount[iIdx - 1];
           iB = pNonZeroCoeffCount[iIdx - 8];
           WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-          WriteBlockResidualCavlc (pBlock, 15, kiA > 0, LUMA_4x4, iC, pBs);
+          if (WriteBlockResidualCavlc (pBlock, 15, kiA > 0, LUMA_4x4, iC, pBs))
+            return ENC_RETURN_VLCOVERFLOWFOUND;
 
           iA = kiA;
           iB = pNonZeroCoeffCount[iIdx - 7];
           WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-          WriteBlockResidualCavlc (pBlock + 16, 15, kiB > 0, LUMA_4x4, iC, pBs);
+          if (WriteBlockResidualCavlc (pBlock + 16, 15, kiB > 0, LUMA_4x4, iC, pBs))
+            return ENC_RETURN_VLCOVERFLOWFOUND;
 
           iA = pNonZeroCoeffCount[iIdx + 7];
           iB = kiA;
           WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-          WriteBlockResidualCavlc (pBlock + 32, 15, kiC > 0, LUMA_4x4, iC, pBs);
+          if (WriteBlockResidualCavlc (pBlock + 32, 15, kiC > 0, LUMA_4x4, iC, pBs))
+            return ENC_RETURN_VLCOVERFLOWFOUND;
 
           iA = kiC;
           iB = kiB;
           WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-          WriteBlockResidualCavlc (pBlock + 48, 15, kiD > 0, LUMA_4x4, iC, pBs);
+          if (WriteBlockResidualCavlc (pBlock + 48, 15, kiD > 0, LUMA_4x4, iC, pBs))
+            return ENC_RETURN_VLCOVERFLOWFOUND;
         }
         pBlock += 64;
       }
@@ -343,7 +348,8 @@
         iA = pNonZeroCoeffCount[iIdx - 1];
         iB = pNonZeroCoeffCount[iIdx - 8];
         WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-        WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs);
+        if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs))
+          return ENC_RETURN_VLCOVERFLOWFOUND;
         pBlock += 16;
       }
 
@@ -354,7 +360,8 @@
         iA = pNonZeroCoeffCount[iIdx - 1];
         iB = pNonZeroCoeffCount[iIdx - 8];
         WELS_NON_ZERO_COUNT_AVERAGE (iC, iA, iB);
-        WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs);
+        if (WriteBlockResidualCavlc (pBlock, 14, pNonZeroCoeffCount[iIdx] > 0, CHROMA_AC, iC, pBs))
+          return ENC_RETURN_VLCOVERFLOWFOUND;
         pBlock += 16;
       }
     }