shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix ubsan warnings: vp9/encoder/vp9_encoder.c

BUG=webm:1219

Change-Id: I433a688fbbc87094afa888c1e7d49f3eefc0e765

--- a/vp9/encoder/vp9_encoder.c

+++ b/vp9/encoder/vp9_encoder.c

@@ -2253,7 +2253,7 @@

 static void encoder_highbd_variance64(const uint8_t *a8, int  a_stride,

                                       const uint8_t *b8, int  b_stride,

                                       int w, int h, uint64_t *sse,

-                                      uint64_t *sum) {

+                                      int64_t *sum) {

   int i, j;

   uint16_t *a = CONVERT_TO_SHORTPTR(a8);

@@ -2277,7 +2277,7 @@

                                       int w, int h,

                                       unsigned int *sse, int *sum) {

   uint64_t sse_long = 0;

-  uint64_t sum_long = 0;

+  int64_t sum_long = 0;

   encoder_highbd_variance64(a8, a_stride, b8, b_stride, w, h,

                             &sse_long, &sum_long);

   *sse = (unsigned int)sse_long;