shithub: libvpx

Info  •  Files  •  Log  •  Branches

Merge changes I1f1edeaa,I89313cac

* changes:
  quantize: silence unsigned overflow warning
  quantize test: quiet overflow warning

--- a/test/vp9_quantize_test.cc

+++ b/test/vp9_quantize_test.cc

@@ -210,9 +210,9 @@

     // Two random entries

     coeff.Set(0);

     coeff.TopLeftPixel()[rnd(count)] =

-        rnd.RandRange(max_value_ * 2) - max_value_;

+        static_cast<int>(rnd.RandRange(max_value_ * 2)) - max_value_;

     coeff.TopLeftPixel()[rnd(count)] =

-        rnd.RandRange(max_value_ * 2) - max_value_;

+        static_cast<int>(rnd.RandRange(max_value_ * 2)) - max_value_;

     GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_,

                          quant_shift_ptr_, dequant_ptr_);

--- a/vpx_dsp/quantize.c

+++ b/vpx_dsp/quantize.c

@@ -203,7 +203,9 @@

         const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;

         const uint32_t abs_qcoeff =

             (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);

-        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);

+        // Restoring the sign triggers unsigned overflow warnings with negative

+        // values because the result of the xor operation is unsigned.

+        qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;

         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];

         if (abs_qcoeff) eob = i;

       }

@@ -310,7 +312,7 @@

       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;

       const uint32_t abs_qcoeff =

           (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);

-      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);

+      qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;

       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;

       if (abs_qcoeff) eob = idx_arr[i];

     }