ref: 8648a64c83b5660eae774d5d41f34fc014ba7b4b
parent: 4ec3f4d83fb7616f5ba6b00481b32f819c9bc2b4
parent: e28bc782048a9f21deb634e00ddc2d4343b3cd33
author: Jerome Jiang <[email protected]>
date: Fri Jun 15 20:12:44 EDT 2018
Merge "VP9 HBD: Fix integer overflow problem in variance calc."
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -378,7 +378,8 @@
static void get_variance(var *v) {
v->variance =
(int)(256 * (v->sum_square_error -
- ((v->sum_error * v->sum_error) >> v->log2_count)) >>
+ (uint32_t)(((int64_t)v->sum_error * v->sum_error) >>
+ v->log2_count)) >>
v->log2_count);
}