shithub: libvpx

Download patch

ref: 12344f26975c7ecd9ce2e1c75aa98dd068711ed7
parent: 04db245a2f98b6c1a53bd5c37a52e547b70da688
author: Jingning Han <[email protected]>
date: Mon Oct 6 06:18:17 EDT 2014 

Add range check in inverse ADST 16x16

Bit-stream clarification related to Issue 868.

Change-Id: I92a7bc5b7782c9ea5c3f6cceec761742183c9514


--- a/test/dct16x16_test.cc
+++ b/test/dct16x16_test.cc
@@ -443,7 +443,7 @@
 
   void RunQuantCheck(int dc_thred, int ac_thred) {
     ACMRandom rnd(ACMRandom::DeterministicSeed());
-    const int count_test_block = 1000;
+    const int count_test_block = 100000;
     DECLARE_ALIGNED_ARRAY(16, int16_t, input_block, kNumCoeffs);
     DECLARE_ALIGNED_ARRAY(16, int16_t, input_extreme_block, kNumCoeffs);
     DECLARE_ALIGNED_ARRAY(16, tran_low_t, output_ref_block, kNumCoeffs);
@@ -700,7 +700,7 @@
 TEST_P(Trans16x16HT, QuantCheck) {
   // The encoder skips any non-DC intra prediction modes,
   // when the quantization step size goes beyond 988.
-  RunQuantCheck(549, 988);
+  RunQuantCheck(429, 729);
 }
 
 using std::tr1::make_tuple;
--- a/vp9/common/vp9_idct.c
+++ b/vp9/common/vp9_idct.c
@@ -794,18 +794,18 @@
   s14 = - x14 * cospi_24_64 + x15 * cospi_8_64;
   s15 =   x14 * cospi_8_64  + x15 * cospi_24_64;
 
-  x0 = WRAPLOW(s0 + s2, 8);
-  x1 = WRAPLOW(s1 + s3, 8);
-  x2 = WRAPLOW(s0 - s2, 8);
-  x3 = WRAPLOW(s1 - s3, 8);
+  x0 = WRAPLOW(check_range(s0 + s2), 8);
+  x1 = WRAPLOW(check_range(s1 + s3), 8);
+  x2 = WRAPLOW(check_range(s0 - s2), 8);
+  x3 = WRAPLOW(check_range(s1 - s3), 8);
   x4 = WRAPLOW(dct_const_round_shift(s4 + s6), 8);
   x5 = WRAPLOW(dct_const_round_shift(s5 + s7), 8);
   x6 = WRAPLOW(dct_const_round_shift(s4 - s6), 8);
   x7 = WRAPLOW(dct_const_round_shift(s5 - s7), 8);
-  x8 = WRAPLOW(s8 + s10, 8);
-  x9 = WRAPLOW(s9 + s11, 8);
-  x10 = WRAPLOW(s8 - s10, 8);
-  x11 = WRAPLOW(s9 - s11, 8);
+  x8 = WRAPLOW(check_range(s8 + s10), 8);
+  x9 = WRAPLOW(check_range(s9 + s11), 8);
+  x10 = WRAPLOW(check_range(s8 - s10), 8);
+  x11 = WRAPLOW(check_range(s9 - s11), 8);
   x12 = WRAPLOW(dct_const_round_shift(s12 + s14), 8);
   x13 = WRAPLOW(dct_const_round_shift(s13 + s15), 8);
   x14 = WRAPLOW(dct_const_round_shift(s12 - s14), 8);
--- a/vp9/common/vp9_idct.h
+++ b/vp9/common/vp9_idct.h
@@ -77,8 +77,7 @@
 static const tran_high_t sinpi_3_9 = 13377;
 static const tran_high_t sinpi_4_9 = 15212;
 
-static INLINE tran_low_t dct_const_round_shift(tran_high_t input) {
-  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
+static INLINE tran_low_t check_range(tran_high_t input) {
 #if CONFIG_VP9_HIGHBITDEPTH
   // For valid highbitdepth VP9 streams, intermediate stage coefficients will
   // stay within the ranges:
@@ -92,10 +91,15 @@
   // this range for every intermediate coefficient can burdensome for a decoder,
   // therefore the following assertion is only enabled when configured with
   // --enable-coefficient-range-checking.
-  assert(INT16_MIN <= rv);
-  assert(rv <= INT16_MAX);
+  assert(INT16_MIN <= input);
+  assert(input <= INT16_MAX);
 #endif
-  return (tran_low_t)rv;
+  return (tran_low_t)input;
+}
+
+static INLINE tran_low_t dct_const_round_shift(tran_high_t input) {
+  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
+  return check_range(rv);
 }
 
 typedef void (*transform_1d)(const tran_low_t*, tran_low_t*);