shithub: libvpx

Info  •  Files  •  Log  •  Branches

Add feature score for each block

The feature score is used to indicate whether a block's mv is reliable
or not.
Now we use Harris Corner Detector method to compute the score.

Change-Id: Ibbe7a1c1f3391d0bf4b03307eaabb5cc3cfb1360

--- a/tools/non_greedy_mv/non_greedy_mv.py

+++ b/tools/non_greedy_mv/non_greedy_mv.py

@@ -84,6 +84,14 @@

   return rgb / 255.

+def read_feature_score(fp, mv_rows, mv_cols):

+  line = fp.readline()

+  word_ls = line.split()

+  feature_score = np.array([float(v) for v in word_ls])

+  feature_score = feature_score.reshape(mv_rows, mv_cols)

+  return feature_score

+

+

 def read_frame_dpl_stats(fp):

   line = fp.readline()

   word_ls = line.split()

@@ -105,12 +113,13 @@

     mv_ls.append([col, row, mv_col, mv_row])

   mv_ls = np.array(mv_ls)

   img = yuv_to_rgb(read_frame(fp))

+  feature_score = read_feature_score(fp, mv_rows, mv_cols)

   ref = None

   line = fp.readline()

   word_ls = line.split()

   if int(word_ls[1]):

     ref = yuv_to_rgb(read_frame(fp))

-  return frame_idx, mv_ls, img, ref, bs

+  return frame_idx, mv_ls, img, ref, bs, feature_score

 def read_dpl_stats_file(filename, frame_num=0):

@@ -131,8 +140,8 @@

 if __name__ == '__main__':

   filename = sys.argv[1]

   data_ls = read_dpl_stats_file(filename, frame_num=5)

-  for frame_idx, mv_ls, img, ref, bs in data_ls:

-    fig, axes = plt.subplots(1, 2)

+  for frame_idx, mv_ls, img, ref, bs, feature_score in data_ls:

+    fig, axes = plt.subplots(1, 3)

     axes[0].imshow(img)

     draw_mv_ls(axes[0], mv_ls)

@@ -148,6 +157,8 @@

       #axes[1].grid(color='k', linestyle='-')

       axes[1].set_ylim(ref.shape[0], 0)

       axes[1].set_xlim(0, ref.shape[1])

+

+    axes[2].imshow(feature_score)

     plt.show()

     print frame_idx, len(mv_ls)

--- a/vp9/encoder/vp9_encoder.c

+++ b/vp9/encoder/vp9_encoder.c

@@ -5730,6 +5730,7 @@

         tpl_ptr->sse_arr[rf_idx] = src_stats->sse_arr[rf_idx];

         tpl_ptr->mv_arr[rf_idx].as_int = src_stats->mv_arr[rf_idx].as_int;

       }

+      tpl_ptr->feature_score = src_stats->feature_score;

 #endif

       tpl_ptr->intra_cost = intra_cost;

       tpl_ptr->inter_cost = inter_cost;

@@ -5845,6 +5846,31 @@

   }

 }

+#if CONFIG_NON_GREEDY_MV

+double get_feature_score(uint8_t *buf, ptrdiff_t stride, int rows, int cols) {

+  double IxIx = 0;

+  double IxIy = 0;

+  double IyIy = 0;

+  double score;

+  int r, c;

+  vpx_clear_system_state();

+  for (r = 0; r + 1 < rows; ++r) {

+    for (c = 0; c + 1 < cols; ++c) {

+      int diff_x = buf[r * stride + c] - buf[r * stride + c + 1];

+      int diff_y = buf[r * stride + c] - buf[(r + 1) * stride + c];

+      IxIx += diff_x * diff_x;

+      IxIy += diff_x * diff_y;

+      IyIy += diff_y * diff_y;

+    }

+  }

+  IxIx /= (rows - 1) * (cols - 1);

+  IxIy /= (rows - 1) * (cols - 1);

+  IyIy /= (rows - 1) * (cols - 1);

+  score = IxIx * IyIy - IxIy * IxIy - 0.04 * (IxIx + IyIy) * (IxIx + IyIy);

+  return score;

+}

+#endif

+

 void mode_estimation(VP9_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *xd,

                      struct scale_factors *sf, GF_PICTURE *gf_picture,

                      int frame_idx, int16_t *src_diff, tran_low_t *coeff,

@@ -5916,6 +5942,11 @@

   x->mv_limits.col_max =

       ((cm->mi_cols - 1 - mi_col) * MI_SIZE) + (17 - 2 * VP9_INTERP_EXTEND);

+#if CONFIG_NON_GREEDY_MV

+  tpl_stats->feature_score = get_feature_score(

+      xd->cur_buf->y_buffer + mb_y_offset, xd->cur_buf->y_stride, bw, bh);

+#endif

+

   for (rf_idx = 0; rf_idx < 3; ++rf_idx) {

     int_mv mv;

     if (ref_frame[rf_idx] == NULL) {

@@ -6133,6 +6164,17 @@

     }

     dump_frame_buf(gf_picture[frame_idx].frame);

+

+    for (mi_row = 0; mi_row < cm->mi_rows; ++mi_row) {

+      for (mi_col = 0; mi_col < cm->mi_cols; ++mi_col) {

+        if ((mi_row % mi_height) == 0 && (mi_col % mi_width) == 0) {

+          const TplDepStats *tpl_ptr =

+              &tpl_frame->tpl_stats_ptr[mi_row * tpl_frame->stride + mi_col];

+          printf("%f ", tpl_ptr->feature_score);

+        }

+      }

+    }

+    printf("\n");

     rf_idx = gf_picture[frame_idx].ref_frame[idx];

     printf("has_ref %d\n", rf_idx != -1);

--- a/vp9/encoder/vp9_encoder.h

+++ b/vp9/encoder/vp9_encoder.h

@@ -295,6 +295,7 @@

   int64_t recon_error_arr[3];

   int64_t sse_arr[3];

   int_mv mv_arr[3];

+  double feature_score;

 #endif

 } TplDepStats;