shithub: libvpx

Download patch

ref: c2217f22381fa869d1aed2d3543edff70cce88c3
parent: 333345cd26cb459f5b1c4f26cd3174b0838ba1bc
author: Jingning Han <[email protected]>
date: Mon Nov 4 13:45:45 EST 2013 

Avoid mv cost check for invalid mv values

The compound inter prediction could potentially run with initial
motion vectors of invalid value and check the mv_cost, which triggers
overheap read. This commit resolves this issue by forcing a motion
vector value check for compound inter modes of both superblock and
sub8x8 block sizes.

Change-Id: I4f4fc19ce83c8272782bc382f12c82a3f03212fc


--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -1872,12 +1872,14 @@
           mi_buf_restore(x, orig_src, orig_pre);
         }
 
-        if (has_second_rf && this_mode == NEWMV &&
-            mbmi->interp_filter == EIGHTTAP) {
+        if (has_second_rf) {
           if (seg_mvs[i][mbmi->ref_frame[1]].as_int == INVALID_MV ||
               seg_mvs[i][mbmi->ref_frame[0]].as_int == INVALID_MV)
             continue;
+        }
 
+        if (has_second_rf && this_mode == NEWMV &&
+            mbmi->interp_filter == EIGHTTAP) {
           // adjust src pointers
           mi_buf_shift(x, i);
           if (cpi->sf.comp_inter_joint_search_thresh <= bsize) {
@@ -2661,6 +2663,12 @@
   int orig_dst_stride[MAX_MB_PLANE];
   int rs = 0;
 
+  if (is_comp_pred) {
+    if (frame_mv[refs[0]].as_int == INVALID_MV ||
+        frame_mv[refs[1]].as_int == INVALID_MV)
+      return INT64_MAX;
+  }
+
   if (this_mode == NEWMV) {
     int rate_mv;
     if (is_comp_pred) {
@@ -2679,9 +2687,6 @@
                                    &mbmi->ref_mvs[refs[1]][0].as_mv,
                                    x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
       }
-      if (frame_mv[refs[0]].as_int == INVALID_MV ||
-          frame_mv[refs[1]].as_int == INVALID_MV)
-        return INT64_MAX;
       *rate2 += rate_mv;
     } else {
       int_mv tmp_mv;