shithub: libvpx

Download patch

ref: d37c78819a02ee79eb129cd85e681080c2350e8c
parent: e7fc39fdf5c16d3559045fb200c3cb00c4b2bc77
parent: b1fcd1751e2b63efbb9f6b4960a42bd757e0c47e
author: Alex Converse <[email protected]>
date: Fri Nov 20 12:41:58 EST 2015 

Merge "Fix unsigned overflow in rd_variance_adjustment."


--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -3017,9 +3017,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }
 
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2831,9 +2831,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }