shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge "Detect invalid highbd iht input"

--- a/vp9/common/vp9_idct.c

+++ b/vp9/common/vp9_idct.c

@@ -204,6 +204,18 @@

 }

 #if CONFIG_VP9_HIGHBITDEPTH

+

+// 12 signal input bits + 7 forward transform amplify bits + 1 bit

+// for contingency in rounding and quantizing

+#define VALID_IHT_MAGNITUDE_RANGE (1 << 20)

+

+static INLINE int detect_invalid_iht_input(const tran_low_t *input, int size) {

+  int i;

+  for (i = 0; i < size; ++i)

+    if (abs(input[i]) >= VALID_IHT_MAGNITUDE_RANGE) return 1;

+  return 0;

+}

+

 void vp9_highbd_iht4x4_16_add_c(const tran_low_t *input, uint8_t *dest8,

                                 int stride, int tx_type, int bd) {

   const highbd_transform_2d IHT_4[] = {

@@ -219,6 +231,13 @@

   tran_low_t *outptr = out;

   tran_low_t temp_in[4], temp_out[4];

+  if (detect_invalid_iht_input(input, 16)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd iht input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    return;

+  }

+

   // Inverse transform row vectors.

   for (i = 0; i < 4; ++i) {

     IHT_4[tx_type].rows(input, outptr, bd);

@@ -253,6 +272,13 @@

   const highbd_transform_2d ht = HIGH_IHT_8[tx_type];

   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);

+  if (detect_invalid_iht_input(input, 64)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd iht input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    return;

+  }

+

   // Inverse transform row vectors.

   for (i = 0; i < 8; ++i) {

     ht.rows(input, outptr, bd);

@@ -286,6 +312,13 @@

   tran_low_t temp_in[16], temp_out[16];

   const highbd_transform_2d ht = HIGH_IHT_16[tx_type];

   uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);

+

+  if (detect_invalid_iht_input(input, 256)) {

+#if CONFIG_COEFFICIENT_RANGE_CHECKING

+    assert(0 && "invalid highbd iht input");

+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING

+    return;

+  }

   // Rows

   for (i = 0; i < 16; ++i) {