shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge "Fix intermediate height in convolve"

--- a/vp9/common/vp9_convolve.c

+++ b/vp9/common/vp9_convolve.c

@@ -217,12 +217,13 @@

    * h == 64, taps == 8.

    */

   uint8_t temp[64 * 135];

-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;

+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;

   assert(w <= 64);

   assert(h <= 64);

   assert(taps <= 8);

   assert(y_step_q4 <= 32);

+  assert(x_step_q4 <= 32);

   if (intermediate_height < h)

     intermediate_height = h;

@@ -246,12 +247,13 @@

    * h == 64, taps == 8.

    */

   uint8_t temp[64 * 135];

-  int intermediate_height = ((h * y_step_q4) >> 4) + taps - 1;

+  int intermediate_height = MAX(((h * y_step_q4) >> 4), 1) + taps - 1;

   assert(w <= 64);

   assert(h <= 64);

   assert(taps <= 8);

   assert(y_step_q4 <= 32);

+  assert(x_step_q4 <= 32);

   if (intermediate_height < h)

     intermediate_height = h;