shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix a potential invalid memory access in non-RD coding flow

This commit fixes a potential out-of-boundary memory access due to
the use of reuse_inter_pred_sby in the non-RD coding flow. It
resolves the corresponding asan error.

Change-Id: Iff605f5921230966990013541cd855d698810922

--- a/test/cpu_speed_test.cc

+++ b/test/cpu_speed_test.cc

@@ -139,5 +139,5 @@

     CpuSpeedTest,

     ::testing::Values(::libvpx_test::kTwoPassGood, ::libvpx_test::kOnePassGood,

                       ::libvpx_test::kRealTime),

-    ::testing::Range(0, 8));

+    ::testing::Range(0, 9));

 }  // namespace

--- a/vp9/encoder/vp9_pickmode.c

+++ b/vp9/encoder/vp9_pickmode.c

@@ -685,7 +685,7 @@

     int rate2 = 0;

     int64_t dist2 = 0;

-    const int dst_stride = pd->dst.stride;

+    const int dst_stride = cpi->sf.reuse_inter_pred_sby ? bw : pd->dst.stride;

     const int src_stride = p->src.stride;

     int block_idx = 0;

--- a/vp9/encoder/vp9_speed_features.c

+++ b/vp9/encoder/vp9_speed_features.c

@@ -296,6 +296,7 @@

   }

   if (speed >= 8) {

     int i;

+    sf->max_intra_bsize = BLOCK_32X32;

     for (i = 0; i < BLOCK_SIZES; ++i)

       sf->inter_mode_mask[i] = INTER_NEAREST;

   }