shithub: libvpx

Info  •  Files  •  Log  •  Branches

Fix a enc/dec mismatch under CONFIG_MISC_FIXES

The culprit is on the decode side xd->lossless[i] setup was in wrong
location where segment features are not yet decoded.

Also on the encoder side, transform mode was not set consistently
between when tx_mode is selected and how tx_mode is enforced in
tx size selection.

Change-Id: I4c4c32188fda7530cadab9b46d4201f33f7ceca3

--- a/vp10/decoder/decodeframe.c

+++ b/vp10/decoder/decodeframe.c

@@ -1143,24 +1143,13 @@

       vpx_rb_read_inv_signed_literal(rb, CONFIG_MISC_FIXES ? 6 : 4) : 0;

 }

-static void setup_quantization(VP10_COMMON *const cm, MACROBLOCKD *const xd,

+static void setup_quantization(VP10_COMMON *const cm,

                                struct vpx_read_bit_buffer *rb) {

-  int i;

-

   cm->base_qindex = vpx_rb_read_literal(rb, QINDEX_BITS);

   cm->y_dc_delta_q = read_delta_q(rb);

   cm->uv_dc_delta_q = read_delta_q(rb);

   cm->uv_ac_delta_q = read_delta_q(rb);

   cm->dequant_bit_depth = cm->bit_depth;

-  for (i = 0; i < MAX_SEGMENTS; ++i) {

-    const int qindex = CONFIG_MISC_FIXES && cm->seg.enabled ?

-                       vp10_get_qindex(&cm->seg, i, cm->base_qindex) :

-                       cm->base_qindex;

-    xd->lossless[i] = qindex == 0 &&

-                      cm->y_dc_delta_q == 0 &&

-                      cm->uv_dc_delta_q == 0 &&

-                      cm->uv_ac_delta_q == 0;

-  }

 #if CONFIG_VP9_HIGHBITDEPTH

   xd->bd = (int)cm->bit_depth;

@@ -1874,9 +1863,7 @@

 static size_t read_uncompressed_header(VP10Decoder *pbi,

                                        struct vpx_read_bit_buffer *rb) {

   VP10_COMMON *const cm = &pbi->common;

-#if CONFIG_MISC_FIXES

   MACROBLOCKD *const xd = &pbi->mb;

-#endif

   BufferPool *const pool = cm->buffer_pool;

   RefCntBuffer *const frame_bufs = pool->frame_bufs;

   int i, mask, ref_index = 0;

@@ -2104,12 +2091,26 @@

     vp10_setup_past_independence(cm);

   setup_loopfilter(&cm->lf, rb);

-  setup_quantization(cm, &pbi->mb, rb);

+  setup_quantization(cm, rb);

   setup_segmentation(cm, rb);

+

+  {

+    int i;

+    for (i = 0; i < MAX_SEGMENTS; ++i) {

+      const int qindex = CONFIG_MISC_FIXES && cm->seg.enabled ?

+          vp10_get_qindex(&cm->seg, i, cm->base_qindex) :

+          cm->base_qindex;

+      xd->lossless[i] = qindex == 0 &&

+          cm->y_dc_delta_q == 0 &&

+          cm->uv_dc_delta_q == 0 &&

+          cm->uv_ac_delta_q == 0;

+    }

+  }

+

   setup_segmentation_dequant(cm);

 #if CONFIG_MISC_FIXES

-  cm->tx_mode = (!cm->seg.enabled && xd->lossless[0]) ? ONLY_4X4

-                                                      : read_tx_mode(rb);

+  cm->tx_mode = (xd->lossless[0]) ? ONLY_4X4

+                                  : read_tx_mode(rb);

   cm->reference_mode = read_frame_reference_mode(cm, rb);

 #endif

--- a/vp10/encoder/bitstream.c

+++ b/vp10/encoder/bitstream.c

@@ -897,19 +897,18 @@

   if (mode != TX_MODE_SELECT)

     vpx_wb_write_literal(wb, mode, 2);

 }

+#else

+static void write_txfm_mode(TX_MODE mode, struct vpx_writer *wb) {

+  vpx_write_literal(wb, VPXMIN(mode, ALLOW_32X32), 2);

+  if (mode >= ALLOW_32X32)

+    vpx_write_bit(wb, mode == TX_MODE_SELECT);

+}

 #endif

+

 static void update_txfm_probs(VP10_COMMON *cm, vpx_writer *w,

                               FRAME_COUNTS *counts) {

-#if !CONFIG_MISC_FIXES

-  // Mode

-  vpx_write_literal(w, VPXMIN(cm->tx_mode, ALLOW_32X32), 2);

-  if (cm->tx_mode >= ALLOW_32X32)

-    vpx_write_bit(w, cm->tx_mode == TX_MODE_SELECT);

-  // Probabilities

-#endif

-

   if (cm->tx_mode == TX_MODE_SELECT) {

     int i, j;

     unsigned int ct_8x8p[TX_SIZES - 3][2];

@@ -1261,7 +1260,7 @@

   encode_quantization(cm, wb);

   encode_segmentation(cm, xd, wb);

 #if CONFIG_MISC_FIXES

-  if (!cm->seg.enabled && xd->lossless[0])

+  if (xd->lossless[0])

     cm->tx_mode = TX_4X4;

   else

     write_txfm_mode(cm->tx_mode, wb);

@@ -1291,10 +1290,12 @@

   vpx_start_encode(&header_bc, data);

 #if !CONFIG_MISC_FIXES

-  if (cpi->td.mb.e_mbd.lossless[0])

+  if (cpi->td.mb.e_mbd.lossless[0]) {

     cm->tx_mode = TX_4X4;

-  else

+  } else {

+    write_txfm_mode(cm->tx_mode, &header_bc);

     update_txfm_probs(cm, &header_bc, counts);

+  }

 #else

   update_txfm_probs(cm, &header_bc, counts);

 #endif

--- a/vp10/encoder/rdopt.c

+++ b/vp10/encoder/rdopt.c

@@ -725,7 +725,7 @@

   assert(bs == xd->mi[0]->mbmi.sb_type);

-  if (CONFIG_MISC_FIXES && xd->lossless[xd->mi[0]->mbmi.segment_id]) {

+  if (CONFIG_MISC_FIXES && xd->lossless[0]) {

     choose_smallest_tx_size(cpi, x, rate, distortion, skip, ret_sse,

                             ref_best_rd, bs);

   } else if (cpi->sf.tx_size_search_method == USE_LARGESTALL ||