shithub: libvpx

Info  •  Files  •  Log  •  Branches

Scale the distortion mectric with tx size

Properly scale the distortion metric according to the tranfer
function gain of the transform block size.

Change-Id: I8e3539d8936f5db78c1352f902f72ef19fc09ed8

--- a/vp9/encoder/vp9_encoder.c

+++ b/vp9/encoder/vp9_encoder.c

@@ -5720,14 +5720,16 @@

   const scan_order *const scan_order = &vp9_default_scan_orders[tx_size];

   uint16_t eob;

   int pix_num = 1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]];

+  const int shift = tx_size == TX_32X32 ? 0 : 2;

   vp9_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, p->quant_fp,

                         qcoeff, dqcoeff, pd->dequant, &eob, scan_order->scan,

                         scan_order->iscan);

-  *recon_error = vp9_block_error(coeff, dqcoeff, pix_num, sse);

-

+  *recon_error = vp9_block_error(coeff, dqcoeff, pix_num, sse) >> shift;

   *recon_error = VPXMAX(*recon_error, 1);

+

+  *sse = (*sse) >> shift;

   *sse = VPXMAX(*sse, 1);

 }