shithub: libvpx

Download patch

ref: 5d635365bb9166d0207c878f92c98835474c9e11
parent: 0a92dd73198fef0e208466a96acefbfecb9bac0b
parent: 5b073c695ba70c87daa1ff843e57ff59f5e3034d
author: Angie Chiang <[email protected]>
date: Tue Oct 4 12:57:36 EDT 2016 

Merge "Move highbd txfm input range check from 2d iht transform to 1d idct/iadst"


--- a/vp9/common/vp9_idct.c
+++ b/vp9/common/vp9_idct.c
@@ -203,17 +203,6 @@
 
 #if CONFIG_VP9_HIGHBITDEPTH
 
-// 12 signal input bits + 7 forward transform amplify bits + 1 bit
-// for contingency in rounding and quantizing
-#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)
-
-static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {
-  int i;
-  for (i = 0; i < size; ++i)
-    if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;
-  return 0;
-}
-
 void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
                                 int stride, int tx_type, int bd) {
   const highbd_transform_2d IHT_4[] = {
@@ -229,13 +218,6 @@
   tran_low_t *outptr = out;
   tran_low_t temp_in[4], temp_out[4];
 
-  if (detect_invalid_iht_input(input, 16)) {
-#if CONFIG_COEFFICIENT_RANGE_CHECKING
-    assert(0 && "invalid highbd iht input");
-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
-    return;
-  }
-
   // Inverse transform row vectors.
   for (i = 0; i < 4; ++i) {
     IHT_4[tx_type].rows(input, outptr, bd);
@@ -270,13 +252,6 @@
   const highbd_transform_2d ht = HIGH_IHT_8[tx_type];
   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
 
-  if (detect_invalid_iht_input(input, 64)) {
-#if CONFIG_COEFFICIENT_RANGE_CHECKING
-    assert(0 && "invalid highbd iht input");
-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
-    return;
-  }
-
   // Inverse transform row vectors.
   for (i = 0; i < 8; ++i) {
     ht.rows(input, outptr, bd);
@@ -310,13 +285,6 @@
   tran_low_t temp_in[16], temp_out[16];
   const highbd_transform_2d ht = HIGH_IHT_16[tx_type];
   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
-
-  if (detect_invalid_iht_input(input, 256)) {
-#if CONFIG_COEFFICIENT_RANGE_CHECKING
-    assert(0 && "invalid highbd iht input");
-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
-    return;
-  }
 
   // Rows
   for (i = 0; i < 16; ++i) {
--- a/vpx_dsp/inv_txfm.c
+++ b/vpx_dsp/inv_txfm.c
@@ -9,6 +9,7 @@
  */
 
 #include <math.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "./vpx_dsp_rtcd.h"
@@ -1252,6 +1253,19 @@
 }
 
 #if CONFIG_VP9_HIGHBITDEPTH
+
+// 12 signal input bits + 7 2D forward transform amplify bits + 5 1D inverse
+// transform amplify bits + 1 bit for contingency in rounding and quantizing
+#define HIGHBD_VALID_TXFM_MAGNITUDE_RANGE (1 << 25)
+
+static INLINE int detect_invalid_highbd_input(const tran_low_t *input,
+                                              int size) {
+  int i;
+  for (i = 0; i < size; ++i)
+    if (abs(input[i]) >= HIGHBD_VALID_TXFM_MAGNITUDE_RANGE) return 1;
+  return 0;
+}
+
 void vpx_highbd_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,
                                  int stride, int bd) {
   /* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,
@@ -1347,6 +1361,15 @@
   tran_low_t step[4];
   tran_high_t temp1, temp2;
   (void)bd;
+
+  if (detect_invalid_highbd_input(input, 4)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 4);
+    return;
+  }
+
   // stage 1
   temp1 = (input[0] + input[2]) * cospi_16_64;
   temp2 = (input[0] - input[2]) * cospi_16_64;
@@ -1413,6 +1436,15 @@
 void vpx_highbd_idct8_c(const tran_low_t *input, tran_low_t *output, int bd) {
   tran_low_t step1[8], step2[8];
   tran_high_t temp1, temp2;
+
+  if (detect_invalid_highbd_input(input, 8)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 8);
+    return;
+  }
+
   // stage 1
   step1[0] = input[0];
   step1[2] = input[4];
@@ -1498,7 +1530,6 @@
 
 void vpx_highbd_iadst4_c(const tran_low_t *input, tran_low_t *output, int bd) {
   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
-
   tran_low_t x0 = input[0];
   tran_low_t x1 = input[1];
   tran_low_t x2 = input[2];
@@ -1505,6 +1536,14 @@
   tran_low_t x3 = input[3];
   (void)bd;
 
+  if (detect_invalid_highbd_input(input, 4)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 4);
+    return;
+  }
+
   if (!(x0 | x1 | x2 | x3)) {
     memset(output, 0, 4 * sizeof(*output));
     return;
@@ -1536,7 +1575,6 @@
 
 void vpx_highbd_iadst8_c(const tran_low_t *input, tran_low_t *output, int bd) {
   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7;
-
   tran_low_t x0 = input[7];
   tran_low_t x1 = input[0];
   tran_low_t x2 = input[5];
@@ -1547,6 +1585,14 @@
   tran_low_t x7 = input[6];
   (void)bd;
 
+  if (detect_invalid_highbd_input(input, 8)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 8);
+    return;
+  }
+
   if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7)) {
     memset(output, 0, 8 * sizeof(*output));
     return;
@@ -1642,6 +1688,14 @@
   tran_high_t temp1, temp2;
   (void)bd;
 
+  if (detect_invalid_highbd_input(input, 16)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 16);
+    return;
+  }
+
   // stage 1
   step1[0] = input[0 / 2];
   step1[1] = input[16 / 2];
@@ -1832,7 +1886,6 @@
 void vpx_highbd_iadst16_c(const tran_low_t *input, tran_low_t *output, int bd) {
   tran_high_t s0, s1, s2, s3, s4, s5, s6, s7, s8;
   tran_high_t s9, s10, s11, s12, s13, s14, s15;
-
   tran_low_t x0 = input[15];
   tran_low_t x1 = input[0];
   tran_low_t x2 = input[13];
@@ -1851,6 +1904,14 @@
   tran_low_t x15 = input[14];
   (void)bd;
 
+  if (detect_invalid_highbd_input(input, 16)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 16);
+    return;
+  }
+
   if (!(x0 | x1 | x2 | x3 | x4 | x5 | x6 | x7 | x8 | x9 | x10 | x11 | x12 |
         x13 | x14 | x15)) {
     memset(output, 0, 16 * sizeof(*output));
@@ -2047,6 +2108,14 @@
   tran_low_t step1[32], step2[32];
   tran_high_t temp1, temp2;
   (void)bd;
+
+  if (detect_invalid_highbd_input(input, 32)) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+    assert(0 && "invalid highbd txfm input");
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+    memset(output, 0, sizeof(*output) * 32);
+    return;
+  }
 
   // stage 1
   step1[0] = input[0];