shithub: libvpx

Info  •  Files  •  Log  •  Branches

VP9 HBD: Fix integer overflow problem in variance calc.

BUG=webm:1534
Change-Id: I535ac48e3dd2454cc7088c4f9a1e08ea74107da6

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -378,7 +378,8 @@

 static void get_variance(var *v) {

   v->variance =

       (int)(256 * (v->sum_square_error -

-                   ((v->sum_error * v->sum_error) >> v->log2_count)) >>

+                   (uint32_t)(((int64_t)v->sum_error * v->sum_error) >>

+                              v->log2_count)) >>

             v->log2_count);

 }