shithub: libvpx

Download patch

ref: e2d6e37a545e817322cf82c5f3b2264140977fb8
parent: 09aca3089f7d35febb2454ce49c7c95cc22be92f
parent: b47cef056e51751d5d7b1cb50f91a71110294a9c
author: Yaowu Xu <[email protected]>
date: Thu Oct 10 09:38:16 EDT 2013 

Merge "change to avoid out-of-range computation"


--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -3633,10 +3633,17 @@
         // values, which actually are bigger than this_rd itself. This can
         // cause negative best_filter_rd[] values, which is obviously silly.
         // Therefore, if filter_cache < ref, we do an adjusted calculation.
-        if (cpi->rd_filter_cache[i] >= ref)
+        if (cpi->rd_filter_cache[i] >= ref) {
           adj_rd = this_rd + cpi->rd_filter_cache[i] - ref;
-        else  // FIXME(rbultje) do this for comppred also
-          adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref;
+        } else {
+          // FIXME(rbultje) do this for comppsred also
+          //
+          // To prevent out-of-range computation in
+          //    adj_rd = cpi->rd_filter_cache[i] * this_rd / ref
+          // cpi->rd_filter_cache[i] / ref is converted to a 256 based ratio.
+          int tmp = cpi->rd_filter_cache[i] * 256 / ref;
+          adj_rd = (this_rd * tmp) >> 8;
+        }
         best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd);
       }
     }