shithub: libvpx

Download patch

ref: aaa7b444607225a0655be6b97ace8128a176028c
parent: b71807082c10f44c5b80b7c81eaa90a93bd500e3
author: Yaowu Xu <[email protected]>
date: Thu Aug 29 06:26:52 EDT 2013 

Fixed potential overflows

The two arrays are typically initialized to INT64_MAX, if they are not
filled with valid values before the addition, the values can overflow
and lead to wrong results.

Change-Id: I515de22cf3e8f55af4b74bdb2c8eb821a02d3059


--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -1247,7 +1247,7 @@
     }
 
     if (cpi->sf.tx_size_search_method == USE_FULL_RD && this_rd < INT64_MAX) {
-      for (i = 0; i < TX_MODES; i++) {
+      for (i = 0; i < TX_MODES && local_tx_cache[i] < INT64_MAX; i++) {
         const int64_t adj_rd = this_rd + local_tx_cache[i] -
             local_tx_cache[cpi->common.tx_mode];
         if (adj_rd < tx_cache[i]) {
@@ -3816,7 +3816,7 @@
       tx_cache[ALLOW_32X32] = tx_cache[ALLOW_16X16];
     }
     if (!mode_excluded && this_rd != INT64_MAX) {
-      for (i = 0; i < TX_MODES; i++) {
+      for (i = 0; i < TX_MODES && tx_cache[i] < INT64_MAX; i++) {
         int64_t adj_rd = INT64_MAX;
         if (this_mode != I4X4_PRED) {
           adj_rd = this_rd + tx_cache[i] - tx_cache[cm->tx_mode];