shithub: libvpx

Info  •  Files  •  Log  •  Branches

vp9_rdopt: correct size to vpx_sum_squares_2d_i16

the current implementations expect pixel size, not the block type

BUG=webm:1392

Change-Id: Ib91e9f30a1f56e13566b1fb76f089dae9bb50cdc

--- a/vp9/encoder/vp9_rdopt.c

+++ b/vp9/encoder/vp9_rdopt.c

@@ -515,7 +515,8 @@

                                              pd->subsampling_y, blk_row);

   if (tx_bsize == BLOCK_4X4 ||

       (b4x4s_to_right_edge >= tx_4x4_w && b4x4s_to_bottom_edge >= tx_4x4_h)) {

-    sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_bsize);

+    assert(tx_4x4_w == tx_4x4_h);

+    sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, tx_4x4_w << 2);

   } else {

     int r, c;

     int max_r = VPXMIN(b4x4s_to_bottom_edge, tx_4x4_h);

@@ -525,7 +526,8 @@

     for (r = 0; r < max_r; ++r) {

       // Skip visiting the sub blocks that are wholly within the UMV.

       for (c = 0; c < max_c; ++c) {

-        sse += (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, BLOCK_4X4);

+        sse += (int64_t)vpx_sum_squares_2d_i16(

+            diff + r * diff_stride * 4 + c * 4, diff_stride, 4);

       }

     }

   }

--- a/vpx_dsp/x86/sum_squares_sse2.c

+++ b/vpx_dsp/x86/sum_squares_sse2.c

@@ -123,6 +123,7 @@

     return vpx_sum_squares_2d_i16_4x4_sse2(src, stride);

   } else {

     // Generic case

+    assert(size % 8 == 0);

     return vpx_sum_squares_2d_i16_nxn_sse2(src, stride, size);

   }

 }