ref: abb13c0d8e378e500504249620dd069f2c35046f
parent: 1d0dedf75921c894895e6674993ba5fd4516b7e2
author: Johann <[email protected]>
date: Tue Apr 30 08:26:18 EDT 2019
cast ambiguous _mm_set1_epiNN() constants clang 7 integer sanitizer warns on unsigned->signed conversions when the highest bit is 1. BUG=webm:1615 Change-Id: I6381efaff9233254b40cb78f7bcf87090e0ad353
--- a/vp9/encoder/x86/vp9_diamond_search_sad_avx.c
+++ b/vp9/encoder/x86/vp9_diamond_search_sad_avx.c
@@ -160,7 +160,7 @@
}
// The inverse mask indicates which of the MVs are outside
- v_outside_d = _mm_xor_si128(v_inside_d, _mm_set1_epi8(0xff));
+ v_outside_d = _mm_xor_si128(v_inside_d, _mm_set1_epi8((int8_t)0xff));
// Shift right to keep the sign bit clear, we will use this later
// to set the cost to the maximum value.
v_outside_d = _mm_srli_epi32(v_outside_d, 1);
--- a/vp9/encoder/x86/vp9_highbd_block_error_intrin_sse2.c
+++ b/vp9/encoder/x86/vp9_highbd_block_error_intrin_sse2.c
@@ -32,7 +32,7 @@
__m128i mm_dqcoeff2 = _mm_load_si128((const __m128i *)(dqcoeff + i + 4));
// Check if any values require more than 15 bit
max = _mm_set1_epi32(0x3fff);
- min = _mm_set1_epi32(0xffffc000);
+ min = _mm_set1_epi32((int32_t)0xffffc000);
cmp0 = _mm_xor_si128(_mm_cmpgt_epi32(mm_coeff, max),
_mm_cmplt_epi32(mm_coeff, min));
cmp1 = _mm_xor_si128(_mm_cmpgt_epi32(mm_coeff2, max),
--- a/vpx_dsp/x86/fwd_txfm_impl_sse2.h
+++ b/vpx_dsp/x86/fwd_txfm_impl_sse2.h
@@ -93,9 +93,9 @@
#if DCT_HIGH_BIT_DEPTH
// Check inputs small enough to use optimised code
cmp0 = _mm_xor_si128(_mm_cmpgt_epi16(in0, _mm_set1_epi16(0x3ff)),
- _mm_cmplt_epi16(in0, _mm_set1_epi16(0xfc00)));
+ _mm_cmplt_epi16(in0, _mm_set1_epi16((int16_t)0xfc00)));
cmp1 = _mm_xor_si128(_mm_cmpgt_epi16(in1, _mm_set1_epi16(0x3ff)),
- _mm_cmplt_epi16(in1, _mm_set1_epi16(0xfc00)));
+ _mm_cmplt_epi16(in1, _mm_set1_epi16((int16_t)0xfc00)));
test = _mm_movemask_epi8(_mm_or_si128(cmp0, cmp1));
if (test) {vpx_highbd_fdct4x4_c(input, output, stride);
--- a/vpx_dsp/x86/highbd_loopfilter_sse2.c
+++ b/vpx_dsp/x86/highbd_loopfilter_sse2.c
@@ -764,8 +764,8 @@
limit_v = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)limit), zero);
thresh_v = _mm_unpacklo_epi8(_mm_load_si128((const __m128i *)thresh), zero);
t80 = _mm_set1_epi16(0x80);
- tff80 = _mm_set1_epi16(0xff80);
- tffe0 = _mm_set1_epi16(0xffe0);
+ tff80 = _mm_set1_epi16((int16_t)0xff80);
+ tffe0 = _mm_set1_epi16((int16_t)0xffe0);
t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 8);
t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 8);
} else if (bd == 10) {@@ -776,8 +776,8 @@
thresh_v = _mm_slli_epi16(
_mm_unpacklo_epi8(_mm_load_si128((const __m128i *)thresh), zero), 2);
t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), 2);
- tff80 = _mm_slli_epi16(_mm_set1_epi16(0xff80), 2);
- tffe0 = _mm_slli_epi16(_mm_set1_epi16(0xffe0), 2);
+ tff80 = _mm_slli_epi16(_mm_set1_epi16((int16_t)0xff80), 2);
+ tffe0 = _mm_slli_epi16(_mm_set1_epi16((int16_t)0xffe0), 2);
t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 6);
t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 6);
} else { // bd == 12@@ -788,8 +788,8 @@
thresh_v = _mm_slli_epi16(
_mm_unpacklo_epi8(_mm_load_si128((const __m128i *)thresh), zero), 4);
t80 = _mm_slli_epi16(_mm_set1_epi16(0x80), 4);
- tff80 = _mm_slli_epi16(_mm_set1_epi16(0xff80), 4);
- tffe0 = _mm_slli_epi16(_mm_set1_epi16(0xffe0), 4);
+ tff80 = _mm_slli_epi16(_mm_set1_epi16((int16_t)0xff80), 4);
+ tffe0 = _mm_slli_epi16(_mm_set1_epi16((int16_t)0xffe0), 4);
t1f = _mm_srli_epi16(_mm_set1_epi16(0x1fff), 4);
t7f = _mm_srli_epi16(_mm_set1_epi16(0x7fff), 4);
}
--- a/vpx_dsp/x86/loopfilter_avx2.c
+++ b/vpx_dsp/x86/loopfilter_avx2.c
@@ -52,7 +52,7 @@
abs_p1p0 =
_mm_or_si128(_mm_subs_epu8(q1p1, q0p0), _mm_subs_epu8(q0p0, q1p1));
abs_q1q0 = _mm_srli_si128(abs_p1p0, 8);
- fe = _mm_set1_epi8(0xfe);
+ fe = _mm_set1_epi8((int8_t)0xfe);
ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
abs_p0q0 =
_mm_or_si128(_mm_subs_epu8(q0p0, p0q0), _mm_subs_epu8(p0q0, q0p0));
@@ -84,7 +84,7 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
const __m128i t1 = _mm_set1_epi16(0x1);
__m128i qs1ps1 = _mm_xor_si128(q1p1, t80);
__m128i qs0ps0 = _mm_xor_si128(q0p0, t80);
@@ -423,7 +423,7 @@
_mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1));
const __m128i abs_q1q0 =
_mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1));
- const __m128i fe = _mm_set1_epi8(0xfe);
+ const __m128i fe = _mm_set1_epi8((int8_t)0xfe);
const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
__m128i abs_p0q0 =
_mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0));
@@ -458,8 +458,8 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
- const __m128i te0 = _mm_set1_epi8(0xe0);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
+ const __m128i te0 = _mm_set1_epi8((int8_t)0xe0);
const __m128i t1f = _mm_set1_epi8(0x1f);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i t7f = _mm_set1_epi8(0x7f);
--- a/vpx_dsp/x86/loopfilter_sse2.c
+++ b/vpx_dsp/x86/loopfilter_sse2.c
@@ -61,7 +61,7 @@
do { \const __m128i t3t4 = \
_mm_set_epi8(3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4); \
- const __m128i t80 = _mm_set1_epi8(0x80); \
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80); \
__m128i filter, filter2filter1, work; \
\
ps1ps0 = _mm_xor_si128(p1p0, t80); /* ^ 0x80 */ \
@@ -263,7 +263,7 @@
__m128i abs_p1q1, abs_p0q0, abs_q1q0, fe, ff, work;
abs_p1p0 = abs_diff(q1p1, q0p0);
abs_q1q0 = _mm_srli_si128(abs_p1p0, 8);
- fe = _mm_set1_epi8(0xfe);
+ fe = _mm_set1_epi8((int8_t)0xfe);
ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
abs_p0q0 = abs_diff(q0p0, p0q0);
abs_p1q1 = abs_diff(q1p1, p1q1);
@@ -291,7 +291,7 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
const __m128i t1 = _mm_set1_epi16(0x1);
__m128i qs1ps1 = _mm_xor_si128(q1p1, t80);
__m128i qs0ps0 = _mm_xor_si128(q0p0, t80);
@@ -628,7 +628,7 @@
{const __m128i abs_p1p0 = abs_diff(p1, p0);
const __m128i abs_q1q0 = abs_diff(q1, q0);
- const __m128i fe = _mm_set1_epi8(0xfe);
+ const __m128i fe = _mm_set1_epi8((int8_t)0xfe);
const __m128i ff = _mm_cmpeq_epi8(zero, zero);
__m128i abs_p0q0 = abs_diff(p0, q0);
__m128i abs_p1q1 = abs_diff(p1, q1);
@@ -677,8 +677,8 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
- const __m128i te0 = _mm_set1_epi8(0xe0);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
+ const __m128i te0 = _mm_set1_epi8((int8_t)0xe0);
const __m128i t1f = _mm_set1_epi8(0x1f);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i t7f = _mm_set1_epi8(0x7f);
@@ -954,7 +954,7 @@
{// filter_mask and hev_mask
const __m128i one = _mm_set1_epi8(1);
- const __m128i fe = _mm_set1_epi8(0xfe);
+ const __m128i fe = _mm_set1_epi8((int8_t)0xfe);
const __m128i ff = _mm_cmpeq_epi8(fe, fe);
__m128i abs_p1q1, abs_p0q0, abs_q1q0, abs_p1p0, work;
abs_p1p0 = abs_diff(q1p1, q0p0);
@@ -1054,7 +1054,7 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i ps1 =
_mm_xor_si128(_mm_loadl_epi64((__m128i *)(s - 2 * pitch)), t80);
@@ -1180,7 +1180,7 @@
const __m128i abs_q1q0 =
_mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1));
const __m128i one = _mm_set1_epi8(1);
- const __m128i fe = _mm_set1_epi8(0xfe);
+ const __m128i fe = _mm_set1_epi8((int8_t)0xfe);
const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
__m128i abs_p0q0 =
_mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0));
@@ -1292,8 +1292,8 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
- const __m128i te0 = _mm_set1_epi8(0xe0);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
+ const __m128i te0 = _mm_set1_epi8((int8_t)0xe0);
const __m128i t1f = _mm_set1_epi8(0x1f);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i t7f = _mm_set1_epi8(0x7f);
@@ -1425,7 +1425,7 @@
_mm_or_si128(_mm_subs_epu8(p1, p0), _mm_subs_epu8(p0, p1));
const __m128i abs_q1q0 =
_mm_or_si128(_mm_subs_epu8(q1, q0), _mm_subs_epu8(q0, q1));
- const __m128i fe = _mm_set1_epi8(0xfe);
+ const __m128i fe = _mm_set1_epi8((int8_t)0xfe);
const __m128i ff = _mm_cmpeq_epi8(abs_p1p0, abs_p1p0);
__m128i abs_p0q0 =
_mm_or_si128(_mm_subs_epu8(p0, q0), _mm_subs_epu8(q0, p0));
@@ -1461,8 +1461,8 @@
{const __m128i t4 = _mm_set1_epi8(4);
const __m128i t3 = _mm_set1_epi8(3);
- const __m128i t80 = _mm_set1_epi8(0x80);
- const __m128i te0 = _mm_set1_epi8(0xe0);
+ const __m128i t80 = _mm_set1_epi8((int8_t)0x80);
+ const __m128i te0 = _mm_set1_epi8((int8_t)0xe0);
const __m128i t1f = _mm_set1_epi8(0x1f);
const __m128i t1 = _mm_set1_epi8(0x1);
const __m128i t7f = _mm_set1_epi8(0x7f);