shithub: libvpx

Info  •  Files  •  Log  •  Branches

Remove conversion warnings from hbd shifts

ROUND_POWER_OF_TWO has some poor side effects when used
with [u]int64_t such as doing the shifting in 32bits.

Change-Id: Ic85a19765cd316fb43657cb21c86f35ceb772773

--- a/test/variance_test.cc

+++ b/test/variance_test.cc

@@ -41,6 +41,25 @@

 using ::std::tr1::tuple;

 using libvpx_test::ACMRandom;

+// Truncate high bit depth results by downshifting (with rounding) by:

+// 2 * (bit_depth - 8) for sse

+// (bit_depth - 8) for se

+static void RoundHighBitDepth(int bit_depth, int64_t *se, uint64_t *sse) {

+  switch (bit_depth) {

+    case VPX_BITS_12:

+      *sse = (*sse + 128) >> 8;

+      *se = (*se + 8) >> 4;

+      break;

+    case VPX_BITS_10:

+      *sse = (*sse + 8) >> 4;

+      *se = (*se + 2) >> 2;

+      break;

+    case VPX_BITS_8:

+    default:

+      break;

+  }

+}

+

 static unsigned int mb_ss_ref(const int16_t *src) {

   unsigned int res = 0;

   for (int i = 0; i < 256; ++i) {

@@ -76,10 +95,7 @@

       }

     }

   }

-  if (bit_depth > VPX_BITS_8) {

-    sse = ROUND_POWER_OF_TWO(sse, 2 * (bit_depth - 8));

-    se = ROUND_POWER_OF_TWO(se, bit_depth - 8);

-  }

+  RoundHighBitDepth(bit_depth, &se, &sse);

   *sse_ptr = sse;

   return sse - (((int64_t) se * se) >> (l2w + l2h));

 }

@@ -125,10 +141,7 @@

       }

     }

   }

-  if (bit_depth > VPX_BITS_8) {

-    sse = ROUND_POWER_OF_TWO(sse, 2 * (bit_depth - 8));

-    se = ROUND_POWER_OF_TWO(se, bit_depth - 8);

-  }

+  RoundHighBitDepth(bit_depth, &se, &sse);

   *sse_ptr = sse;

   return sse - (((int64_t) se * se) >> (l2w + l2h));

 }

@@ -496,10 +509,7 @@

       }

     }

   }

-  if (bit_depth > 8) {

-    sse = ROUND_POWER_OF_TWO(sse, 2*(bit_depth-8));

-    se = ROUND_POWER_OF_TWO(se, bit_depth-8);

-  }

+  RoundHighBitDepth(bit_depth, &se, &sse);

   *sse_ptr = sse;

   return sse - (((int64_t) se * se) >> (l2w + l2h));

 }