shithub: libvpx

Info  •  Files  •  Log  •  Branches

Adaptive txfm size selection depending on residual sse/variance

This commit enables an adaptive transform size selection method
for speed -6. It uses largest transform size when the sse is more
than 4 times of variance, i.e., most energy is compacted in the
DC coefficient. Otherwise, use the default TX_8X8. It improves
the compression efficiency for rtc set of speed -6 by 0.8%, no
speed change observed.

Change-Id: Ie6ed1e728ff7bf88ebe940a60811361cdd19969c

--- a/vp9/encoder/vp9_encodeframe.c

+++ b/vp9/encoder/vp9_encodeframe.c

@@ -2390,7 +2390,7 @@

                  rd_opt->tx_select_threshes[frame_type][TX_MODE_SELECT] ?

                      ALLOW_32X32 : TX_MODE_SELECT;

     } else if (cpi->sf.tx_size_search_method == USE_TX_8X8) {

-      return ALLOW_8X8;

+      return TX_MODE_SELECT;

     } else {

       unsigned int total = 0;

       int i;

--- a/vp9/encoder/vp9_pickmode.c

+++ b/vp9/encoder/vp9_pickmode.c

@@ -252,6 +252,17 @@

   else

     x->skip_txfm = 0;

+  if (cpi->common.tx_mode == TX_MODE_SELECT) {

+    if (sse > (var << 2))

+      xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],

+                          tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);

+    else

+      xd->mi[0]->mbmi.tx_size = TX_8X8;

+  } else {

+    xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],

+                         tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);

+  }

+

   vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],

                                dc_quant >> 3, &rate, &dist);

   *out_rate_sum = rate >> 1;

@@ -293,6 +304,8 @@

   struct macroblockd_plane *const pd = &xd->plane[0];

   PREDICTION_MODE this_mode, best_mode = ZEROMV;

   MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;

+  TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],

+                             tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);

   INTERP_FILTER best_pred_filter = EIGHTTAP;

   int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];

   struct buf_2d yv12_mb[4][MAX_MB_PLANE];

@@ -486,6 +499,7 @@

         int64_t pf_dist[3];

         unsigned int pf_var[3];

         unsigned int pf_sse[3];

+        TX_SIZE pf_tx_size[3];

         int64_t best_cost = INT64_MAX;

         INTERP_FILTER best_filter = SWITCHABLE, filter;

         PRED_BUFFER *current_pred = this_mode_pred;

@@ -499,6 +513,7 @@

           cost = RDCOST(x->rdmult, x->rddiv,

                         vp9_get_switchable_rate(cpi) + pf_rate[filter],

                         pf_dist[filter]);

+          pf_tx_size[filter] = mbmi->tx_size;

           if (cost < best_cost) {

             best_filter = filter;

             best_cost = cost;

@@ -523,6 +538,7 @@

           free_pred_buffer(current_pred);

         mbmi->interp_filter = best_filter;

+        mbmi->tx_size = pf_tx_size[mbmi->interp_filter];

         rate = pf_rate[mbmi->interp_filter];

         dist = pf_dist[mbmi->interp_filter];

         var_y = pf_var[mbmi->interp_filter];

@@ -624,6 +640,7 @@

         *returndistortion = dist;

         best_mode = this_mode;

         best_pred_filter = mbmi->interp_filter;

+        best_tx_size = mbmi->tx_size;

         best_ref_frame = ref_frame;

         skip_txfm = x->skip_txfm;

@@ -657,10 +674,11 @@

                       bw, bh);

   }

-  mbmi->mode = best_mode;

+  mbmi->mode          = best_mode;

   mbmi->interp_filter = best_pred_filter;

-  mbmi->ref_frame[0] = best_ref_frame;

-  mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;

+  mbmi->tx_size       = best_tx_size;

+  mbmi->ref_frame[0]  = best_ref_frame;

+  mbmi->mv[0].as_int  = frame_mv[best_mode][best_ref_frame].as_int;

   xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;

   x->skip_txfm = skip_txfm;

@@ -669,7 +687,6 @@

   if (!x->skip && best_rd > inter_mode_thresh &&

       bsize <= cpi->sf.max_intra_bsize) {

     int i, j;

-    const int step   = 1 << mbmi->tx_size;

     const int width  = num_4x4_blocks_wide_lookup[bsize];

     const int height = num_4x4_blocks_high_lookup[bsize];

@@ -679,6 +696,10 @@

     const int src_stride = p->src.stride;

     int block_idx = 0;

+    TX_SIZE tmp_tx_size = MIN(max_txsize_lookup[bsize],

+                              tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);

+    const int step = 1 << tmp_tx_size;

+

     for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {

       if (cpi->sf.reuse_inter_pred_sby) {

         pd->dst.buf = tmp[0].data;

@@ -688,7 +709,7 @@

       for (j = 0; j < height; j += step) {

         for (i = 0; i < width; i += step) {

           vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize),

-                                  mbmi->tx_size, this_mode,

+                                  tmp_tx_size, this_mode,

                                   &p->src.buf[4 * (j * dst_stride + i)],

                                   src_stride,

                                   &pd->dst.buf[4 * (j * dst_stride + i)],

@@ -715,6 +736,7 @@

         *returnrate = rate;

         *returndistortion = dist;

         mbmi->mode = this_mode;

+        mbmi->tx_size = tmp_tx_size;

         mbmi->ref_frame[0] = INTRA_FRAME;

         mbmi->uv_mode = this_mode;

         mbmi->mv[0].as_int = INVALID_MV;

--- a/vp9/encoder/vp9_speed_features.c

+++ b/vp9/encoder/vp9_speed_features.c

@@ -272,7 +272,8 @@

     sf->search_type_check_frequency = 50;

     sf->source_var_thresh = 360;

-    sf->tx_size_search_method = USE_TX_8X8;

+    sf->tx_size_search_method = (cm->frame_type == KEY_FRAME) ?

+        USE_LARGESTALL : USE_TX_8X8;

     sf->max_intra_bsize = BLOCK_8X8;

     // This feature is only enabled when partition search is disabled.