shithub: libvpx

Download patch

ref: b1fcd1751e2b63efbb9f6b4960a42bd757e0c47e
parent: f3f6b6fe3e960959489db2568d9942aeca261daa
author: Alex Converse <[email protected]>
date: Thu Nov 19 09:53:51 EST 2015 

Fix unsigned overflow in rd_variance_adjustment.

Found with clang -fsanitize=integer

Change-Id: I2538e7483cb2d5f06bceecbd3326bdd88bfecfa1


--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -3017,9 +3017,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }
 
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -2831,9 +2831,9 @@
       ? (source_variance - recon_variance)
       : (recon_variance - source_variance);
 
-    var_error = (200 * source_variance * recon_variance) /
-      ((source_variance * source_variance) +
-       (recon_variance * recon_variance));
+    var_error = ((int64_t)200 * source_variance * recon_variance) /
+      (((int64_t)source_variance * source_variance) +
+       ((int64_t)recon_variance * recon_variance));
     var_error = 100 - var_error;
   }