shithub: libvpx

Download patch

ref: 77ed4414d6dc4b047cab10aebf156bef5719513b
parent: 08cb7b5c680e747504f0d518e3e96b0b5467792e
author: Johann <[email protected]>
date: Tue Aug 15 05:48:24 EDT 2017 

quantize: silence unsigned overflow warning

The result of the xor operation is unsigned. If coeff was negative,
this results in an unsigned value - INT_MIN.

Change-Id: I1f1edeaa6de1f4c68b848e8a82a666d390b749f0


--- a/vpx_dsp/quantize.c
+++ b/vpx_dsp/quantize.c
@@ -203,7 +203,9 @@
         const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
         const uint32_t abs_qcoeff =
             (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
-        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
+        // Restoring the sign triggers unsigned overflow warnings with negative
+        // values because the result of the xor operation is unsigned.
+        qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;
         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
         if (abs_qcoeff) eob = i;
       }
@@ -310,7 +312,7 @@
       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
       const uint32_t abs_qcoeff =
           (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
-      qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
+      qcoeff_ptr[rc] = (tran_low_t)(abs_qcoeff ^ coeff_sign) - coeff_sign;
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
       if (abs_qcoeff) eob = idx_arr[i];
     }