shithub: libvpx

Download patch

ref: c7cfe7a9eebd7d0940edacd1b8649343ed477e47
parent: a0befb93e7150f72afe57aa5b7f7df3308690cef
parent: befc36d4a791447e36699d0390dff5205cc6dd3f
author: Debargha Mukherjee <[email protected]>
date: Thu Oct 2 09:51:38 EDT 2014

Merge "Fix invalid memory access in inter prediction (issue 853)."

--- a/vp9/common/vp9_convolve.c
+++ b/vp9/common/vp9_convolve.c
@@ -130,7 +130,8 @@
   // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
   // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
   uint8_t temp[135 * 64];
-  int intermediate_height = (((h - 1) * y_step_q4 + 15) >> 4) + SUBPEL_TAPS;
+  int intermediate_height =
+          (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
 
   assert(w <= 64);
   assert(h <= 64);
@@ -137,9 +138,6 @@
   assert(y_step_q4 <= 32);
   assert(x_step_q4 <= 32);
 
-  if (intermediate_height < h)
-    intermediate_height = h;
-
   convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp, 64,
                  x_filters, x0_q4, x_step_q4, w, intermediate_height);
   convolve_vert(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst, dst_stride,
@@ -407,15 +405,13 @@
   // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
   // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
   uint16_t temp[64 * 135];
-  int intermediate_height = (((h - 1) * y_step_q4 + 15) >> 4) + SUBPEL_TAPS;
+  int intermediate_height =
+          (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
 
   assert(w <= 64);
   assert(h <= 64);
   assert(y_step_q4 <= 32);
   assert(x_step_q4 <= 32);
-
-  if (intermediate_height < h)
-    intermediate_height = h;
 
   high_convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1),
                       src_stride, CONVERT_TO_BYTEPTR(temp), 64,