ref: b6fa0f20565ce10b4724a120ff0c8d6faff0926b
parent: 2d0f3b23a908bb54dbeddf6d1dcc666600470e2e
author: Jorge E. Moreira <[email protected]>
date: Fri May 8 11:23:48 EDT 2020
Temporarily convert to 64 bits to avoid overflows In the vp8_cost_branch function a couple of unsigned int are being multiplied by integer coefficients and added to later be divided by 256. While the end result most likely fits an unsigned int, the intermediary result of multiplying and adding sometimes doesn't (I was able to reproduce it by leaving the encoder running at 60 fps for a while). To avoid the multiplication overflow (which is undefined behavior and causes a wrong result anyways) the calculation is performed using unsigned long long instead and cast to unsigned int for return. Bug: b/154172422 Test: run cuttlefish with webrtc enabled for an hour Change-Id: If7ebbda38b2450a59ed3c99ffbb59dc62431a324
--- a/vp8/encoder/treewriter.h
+++ b/vp8/encoder/treewriter.h
@@ -14,6 +14,8 @@
/* Trees map alphabets into huffman-like codes suitable for an arithmetic
bit coder. Timothy S Murphy 11 October 2004 */
+#include <stdint.h>
+
#include "./vpx_config.h"
#include "vp8/common/treecoder.h"
@@ -48,7 +50,9 @@
vp8_prob p) {/* Imitate existing calculation */
- return ((ct[0] * vp8_cost_zero(p)) + (ct[1] * vp8_cost_one(p))) >> 8;
+ return (unsigned int)(((((uint64_t)ct[0]) * vp8_cost_zero(p)) +
+ (((uint64_t)ct[1]) * vp8_cost_one(p))) >>
+ 8);
}
/* Small functions to write explicit values and tokens, as well as