shithub: libvpx

Download patch

ref: 481a7c5986f17a7c212eb9ab12beac57bcecef66
parent: d9705b200f3c84f9701f29905a77446df4d18140
author: Jingning Han <[email protected]>
date: Fri May 17 06:03:52 EDT 2013 

Fix parameters in 4x8 variance calc

Correct the stride parameter of 4x8 in vp9_sub_pixel_variance4x8_
and vp9_sub_pixel_avg_variance4x8.

Change-Id: I2ca74d4043817503b21737563994270e3b0619ff


--- a/vp9/encoder/vp9_variance_c.c
+++ b/vp9/encoder/vp9_variance_c.c
@@ -873,6 +873,8 @@
                                          int dst_pixels_per_line,
                                          unsigned int *sse) {
   uint16_t fdata3[5 * 8];  // Temp data bufffer used in filtering
+  // FIXME(jingning,rbultje): this temp2 buffer probably doesn't need to be
+  // of this big? same issue appears in all other block size settings.
   uint8_t temp2[20 * 16];
   const int16_t *hfilter, *vfilter;
 
@@ -880,7 +882,7 @@
   vfilter = VP9_BILINEAR_FILTERS_2TAP(yoffset);
 
   var_filter_block2d_bil_first_pass(src_ptr, fdata3, src_pixels_per_line,
-                                    1, 17, 4, hfilter);
+                                    1, 9, 4, hfilter);
   var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 8, 4, vfilter);
 
   return vp9_variance4x8_c(temp2, 4, dst_ptr, dst_pixels_per_line, sse);
@@ -903,7 +905,7 @@
   vfilter = VP9_BILINEAR_FILTERS_2TAP(yoffset);
 
   var_filter_block2d_bil_first_pass(src_ptr, fdata3, src_pixels_per_line,
-                                    1, 17, 4, hfilter);
+                                    1, 9, 4, hfilter);
   var_filter_block2d_bil_second_pass(fdata3, temp2, 4, 4, 8, 4, vfilter);
   comp_avg_pred(temp3, second_pred, 4, 8, temp2, 4);
   return vp9_variance4x8_c(temp3, 4, dst_ptr, dst_pixels_per_line, sse);