ref: 07c48ccfe090625e409be50ff82b6d8e6fc93578
parent: b71245683b727cdfe86f2f29eac830e88e980f8d
author: Yi Luo <[email protected]>
date: Tue Feb 28 11:38:41 EST 2017
Improve idct32x32_34_add SSSE3 intrinsics performance - Split the transform into first half and second half. - Reschedule the instructions to avoid stack spillover. - Function level speed improves ~16%. Change-Id: I166889840d23aa8a273eca00f6fbdae8b4566f35
--- a/vpx_dsp/x86/inv_txfm_ssse3.c
+++ b/vpx_dsp/x86/inv_txfm_ssse3.c
@@ -378,18 +378,71 @@
RECON_AND_STORE(dest + 7 * stride, in7);
}
-static INLINE void idct32_34(const __m128i *in, __m128i *stp1) {+static INLINE void simple_group_butterfly(const __m128i *in, __m128i *out) {+ int i = 0;
+ while (i < 16) {+ out[i] = _mm_add_epi16(in[i], in[31 - i]);
+ out[31 - i] = _mm_sub_epi16(in[i], in[31 - i]);
+ i++;
+ }
+}
+
+static INLINE void butterfly(const __m128i x0, const __m128i x1,
+ const __m128i c0, const __m128i c1, __m128i *y0,
+ __m128i *y1) {+ __m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
- // idct constants for each stage
- const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
- const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
- const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
- const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
- const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
- const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
- const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
- const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
+ u0 = _mm_unpacklo_epi16(x0, x1);
+ u1 = _mm_unpackhi_epi16(x0, x1);
+
+ tmp0 = _mm_madd_epi16(u0, c0);
+ tmp1 = _mm_madd_epi16(u1, c0);
+ tmp2 = _mm_madd_epi16(u0, c1);
+ tmp3 = _mm_madd_epi16(u1, c1);
+
+ tmp0 = _mm_add_epi32(tmp0, rounding);
+ tmp1 = _mm_add_epi32(tmp1, rounding);
+ tmp2 = _mm_add_epi32(tmp2, rounding);
+ tmp3 = _mm_add_epi32(tmp3, rounding);
+
+ tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
+ tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
+ tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
+ tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
+
+ *y0 = _mm_packs_epi32(tmp0, tmp1);
+ *y1 = _mm_packs_epi32(tmp2, tmp3);
+}
+
+static INLINE void butterfly_self(__m128i *x0, __m128i *x1, const __m128i c0,
+ const __m128i c1) {+ __m128i tmp0, tmp1, tmp2, tmp3, u0, u1;
+ const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING);
+
+ u0 = _mm_unpacklo_epi16(*x0, *x1);
+ u1 = _mm_unpackhi_epi16(*x0, *x1);
+
+ tmp0 = _mm_madd_epi16(u0, c0);
+ tmp1 = _mm_madd_epi16(u1, c0);
+ tmp2 = _mm_madd_epi16(u0, c1);
+ tmp3 = _mm_madd_epi16(u1, c1);
+
+ tmp0 = _mm_add_epi32(tmp0, rounding);
+ tmp1 = _mm_add_epi32(tmp1, rounding);
+ tmp2 = _mm_add_epi32(tmp2, rounding);
+ tmp3 = _mm_add_epi32(tmp3, rounding);
+
+ tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
+ tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
+ tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
+ tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
+
+ *x0 = _mm_packs_epi32(tmp0, tmp1);
+ *x1 = _mm_packs_epi32(tmp2, tmp3);
+}
+
+static void idct32_34_first_half(const __m128i *in, __m128i *stp1) {const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
@@ -397,12 +450,6 @@
const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
- const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
- const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
- const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
- const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
- const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
- const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
@@ -412,294 +459,170 @@
const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
- __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7,
- stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15,
- stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23,
- stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31;
- __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
+ __m128i u0, u1, u2, u3, u4, u5, u6, u7;
+ __m128i x0, x1, x4, x5, x6, x7;
+ __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15;
- /* Stage1 */
+ // phase 1
- stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0);
- stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1);
+ // 0, 15
+ u2 = _mm_mulhrs_epi16(in[2], stk2_1); // stp2_15
+ u3 = _mm_mulhrs_epi16(in[6], stk2_7); // stp2_12
+ v15 = _mm_add_epi16(u2, u3);
+ // in[0], in[4]
+ x0 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[0]
+ x7 = _mm_mulhrs_epi16(in[4], stk3_1); // stp1[7]
+ v0 = _mm_add_epi16(x0, x7); // stp2_0
+ stp1[0] = _mm_add_epi16(v0, v15);
+ stp1[15] = _mm_sub_epi16(v0, v15);
- stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6);
- stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7);
+ // in[2], in[6]
+ u0 = _mm_mulhrs_epi16(in[2], stk2_0); // stp2_8
+ u1 = _mm_mulhrs_epi16(in[6], stk2_6); // stp2_11
+ butterfly(u0, u2, stg4_4, stg4_5, &u4, &u5); // stp2_9, stp2_14
+ butterfly(u1, u3, stg4_6, stg4_4, &u6, &u7); // stp2_10, stp2_13
- stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8);
- stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9);
+ v8 = _mm_add_epi16(u0, u1);
+ v9 = _mm_add_epi16(u4, u6);
+ v10 = _mm_sub_epi16(u4, u6);
+ v11 = _mm_sub_epi16(u0, u1);
+ v12 = _mm_sub_epi16(u2, u3);
+ v13 = _mm_sub_epi16(u5, u7);
+ v14 = _mm_add_epi16(u5, u7);
- stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14);
- stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15);
+ butterfly_self(&v10, &v13, stg6_0, stg4_0);
+ butterfly_self(&v11, &v12, stg6_0, stg4_0);
- /* Stage2 */
+ // 1, 14
+ x1 = _mm_mulhrs_epi16(in[0], stk4_0); // stp1[1], stk4_1 = stk4_0
+ // stp1[2] = stp1[0], stp1[3] = stp1[1]
+ x4 = _mm_mulhrs_epi16(in[4], stk3_0); // stp1[4]
+ butterfly(x7, x4, stg4_1, stg4_0, &x5, &x6);
+ v1 = _mm_add_epi16(x1, x6); // stp2_1
+ v2 = _mm_add_epi16(x0, x5); // stp2_2
+ stp1[1] = _mm_add_epi16(v1, v14);
+ stp1[14] = _mm_sub_epi16(v1, v14);
- stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0);
- stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1);
+ stp1[2] = _mm_add_epi16(v2, v13);
+ stp1[13] = _mm_sub_epi16(v2, v13);
- stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6);
- stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7);
+ v3 = _mm_add_epi16(x1, x4); // stp2_3
+ v4 = _mm_sub_epi16(x1, x4); // stp2_4
- /* Stage3 */
- {- const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]);
- const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]);
- const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]);
- const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]);
+ v5 = _mm_sub_epi16(x0, x5); // stp2_5
- const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]);
- const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]);
- const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]);
- const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]);
+ v6 = _mm_sub_epi16(x1, x6); // stp2_6
+ v7 = _mm_sub_epi16(x0, x7); // stp2_7
+ stp1[3] = _mm_add_epi16(v3, v12);
+ stp1[12] = _mm_sub_epi16(v3, v12);
- stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0);
- stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1);
+ stp1[6] = _mm_add_epi16(v6, v9);
+ stp1[9] = _mm_sub_epi16(v6, v9);
- MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4,
- stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18],
- stp1[29])
- MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8,
- stg3_9, stg3_10, stg3_8, stp1[21], stp1[26],
- stp1[22], stp1[25])
- }
+ stp1[7] = _mm_add_epi16(v7, v8);
+ stp1[8] = _mm_sub_epi16(v7, v8);
- /* Stage4 */
- {- const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15);
- const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15);
- const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12);
- const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12);
+ stp1[4] = _mm_add_epi16(v4, v11);
+ stp1[11] = _mm_sub_epi16(v4, v11);
- stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0);
- stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0
- stp1[2] = stp1[0];
- stp1[3] = stp1[1];
+ stp1[5] = _mm_add_epi16(v5, v10);
+ stp1[10] = _mm_sub_epi16(v5, v10);
+}
- MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5,
- stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13)
+static void idct32_34_second_half(const __m128i *in, __m128i *stp1) {+ const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64);
+ const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64);
+ const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64);
+ const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64);
+ const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64);
+ const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64);
+ const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64);
+ const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64);
+ const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64);
+ const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64);
+ const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64);
+ const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64);
+ const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64);
+ const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64);
- stp2_16 = _mm_add_epi16(stp1[16], stp1[19]);
- stp2_17 = _mm_add_epi16(stp1[17], stp1[18]);
- stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]);
- stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]);
- stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]);
- stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]);
- stp2_22 = _mm_add_epi16(stp1[22], stp1[21]);
- stp2_23 = _mm_add_epi16(stp1[23], stp1[20]);
+ const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64);
+ const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64);
+ const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64);
+ const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
- stp2_24 = _mm_add_epi16(stp1[24], stp1[27]);
- stp2_25 = _mm_add_epi16(stp1[25], stp1[26]);
- stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]);
- stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]);
- stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]);
- stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]);
- stp2_30 = _mm_add_epi16(stp1[29], stp1[30]);
- stp2_31 = _mm_add_epi16(stp1[28], stp1[31]);
- }
+ const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64);
+ __m128i v16, v17, v18, v19, v20, v21, v22, v23;
+ __m128i v24, v25, v26, v27, v28, v29, v30, v31;
+ __m128i u16, u17, u18, u19, u20, u21, u22, u23;
+ __m128i u24, u25, u26, u27, u28, u29, u30, u31;
- /* Stage5 */
- {-// Note:
-// #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass
-// SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop
-// to 23198 and -23197, respectively.
-#define AVOID_OVERFLOW (1)
+ v16 = _mm_mulhrs_epi16(in[1], stk1_0);
+ v31 = _mm_mulhrs_epi16(in[1], stk1_1);
-#if AVOID_OVERFLOW
- const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]);
- const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]);
-#endif
- const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29);
- const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29);
+ v19 = _mm_mulhrs_epi16(in[7], stk1_6);
+ v28 = _mm_mulhrs_epi16(in[7], stk1_7);
- const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28);
- const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28);
- const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
- const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
+ v20 = _mm_mulhrs_epi16(in[5], stk1_8);
+ v27 = _mm_mulhrs_epi16(in[5], stk1_9);
- const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
- const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
+ v23 = _mm_mulhrs_epi16(in[3], stk1_14);
+ v24 = _mm_mulhrs_epi16(in[3], stk1_15);
-#if AVOID_OVERFLOW
- tmp0 = _mm_madd_epi16(lo_6_5, stg4_1);
- tmp1 = _mm_madd_epi16(hi_6_5, stg4_1);
- tmp2 = _mm_madd_epi16(lo_6_5, stg4_0);
- tmp3 = _mm_madd_epi16(hi_6_5, stg4_0);
+ butterfly(v16, v31, stg3_4, stg3_5, &v17, &v30);
+ butterfly(v19, v28, stg3_6, stg3_4, &v18, &v29);
+ butterfly(v20, v27, stg3_8, stg3_9, &v21, &v26);
+ butterfly(v23, v24, stg3_10, stg3_8, &v22, &v25);
- tmp0 = _mm_add_epi32(tmp0, rounding);
- tmp1 = _mm_add_epi32(tmp1, rounding);
- tmp2 = _mm_add_epi32(tmp2, rounding);
- tmp3 = _mm_add_epi32(tmp3, rounding);
+ u16 = _mm_add_epi16(v16, v19);
+ u17 = _mm_add_epi16(v17, v18);
+ u18 = _mm_sub_epi16(v17, v18);
+ u19 = _mm_sub_epi16(v16, v19);
+ u20 = _mm_sub_epi16(v23, v20);
+ u21 = _mm_sub_epi16(v22, v21);
+ u22 = _mm_add_epi16(v22, v21);
+ u23 = _mm_add_epi16(v23, v20);
+ u24 = _mm_add_epi16(v24, v27);
+ u27 = _mm_sub_epi16(v24, v27);
+ u25 = _mm_add_epi16(v25, v26);
+ u26 = _mm_sub_epi16(v25, v26);
+ u28 = _mm_sub_epi16(v31, v28);
+ u31 = _mm_add_epi16(v28, v31);
+ u29 = _mm_sub_epi16(v30, v29);
+ u30 = _mm_add_epi16(v29, v30);
- tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS);
- tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS);
- tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS);
- tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS);
+ butterfly_self(&u18, &u29, stg4_4, stg4_5);
+ butterfly_self(&u19, &u28, stg4_4, stg4_5);
+ butterfly_self(&u20, &u27, stg4_6, stg4_4);
+ butterfly_self(&u21, &u26, stg4_6, stg4_4);
- stp1[5] = _mm_packs_epi32(tmp0, tmp1);
- stp1[6] = _mm_packs_epi32(tmp2, tmp3);
-#else
- tmp0 = _mm_sub_epi16(stp1[7], stp1[4]);
- tmp1 = _mm_adds_epi16(stp1[7], stp1[4]);
- stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0);
- stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0);
-#endif
+ stp1[16] = _mm_add_epi16(u16, u23);
+ v23 = _mm_sub_epi16(u16, u23);
- stp1[8] = _mm_add_epi16(stp2_8, stp2_11);
- stp1[9] = _mm_add_epi16(stp2_9, stp2_10);
- stp1[10] = _mm_sub_epi16(stp2_9, stp2_10);
- stp1[11] = _mm_sub_epi16(stp2_8, stp2_11);
- stp1[12] = _mm_sub_epi16(stp2_15, stp2_12);
- stp1[13] = _mm_sub_epi16(stp2_14, stp2_13);
- stp1[14] = _mm_add_epi16(stp2_14, stp2_13);
- stp1[15] = _mm_add_epi16(stp2_15, stp2_12);
+ stp1[17] = _mm_add_epi16(u17, u22);
+ v22 = _mm_sub_epi16(u17, u22);
- MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4,
- stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19],
- stp1[28])
- MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6,
- stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21],
- stp1[26])
+ stp1[18] = _mm_add_epi16(u18, u21);
+ v21 = _mm_sub_epi16(u18, u21);
- stp1[16] = stp2_16;
- stp1[17] = stp2_17;
- stp1[22] = stp2_22;
- stp1[23] = stp2_23;
- stp1[24] = stp2_24;
- stp1[25] = stp2_25;
- stp1[30] = stp2_30;
- stp1[31] = stp2_31;
- }
+ stp1[19] = _mm_add_epi16(u19, u20);
+ v20 = _mm_sub_epi16(u19, u20);
- /* Stage6 */
- {-#if AVOID_OVERFLOW
- const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]);
- const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]);
- const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]);
- const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]);
-#endif
+ v24 = _mm_sub_epi16(u31, u24);
+ stp1[31] = _mm_add_epi16(u24, u31);
- stp2_0 = _mm_add_epi16(stp1[0], stp1[7]);
- stp2_1 = _mm_add_epi16(stp1[1], stp1[6]);
- stp2_2 = _mm_add_epi16(stp1[2], stp1[5]);
- stp2_3 = _mm_add_epi16(stp1[3], stp1[4]);
- stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]);
- stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]);
- stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]);
- stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]);
+ v25 = _mm_sub_epi16(u30, u25);
+ stp1[30] = _mm_add_epi16(u25, u30);
- stp2_8 = stp1[8];
- stp2_9 = stp1[9];
- stp2_14 = stp1[14];
- stp2_15 = stp1[15];
+ v26 = _mm_sub_epi16(u29, u26);
+ stp1[29] = _mm_add_epi16(u26, u29);
-#if AVOID_OVERFLOW
- MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0,
- stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11,
- stp2_12)
-#else
- tmp0 = _mm_add_epi16(stp1[10], stp1[13]);
- tmp1 = _mm_sub_epi16(stp1[13], stp1[10]);
- tmp2 = _mm_add_epi16(stp1[11], stp1[12]);
- tmp3 = _mm_sub_epi16(stp1[12], stp1[11]);
+ v27 = _mm_sub_epi16(u28, u27);
+ stp1[28] = _mm_add_epi16(u27, u28);
- stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0);
- stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0);
- stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0);
- stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0);
-
-#endif
-
- stp2_16 = _mm_add_epi16(stp1[16], stp1[23]);
- stp2_17 = _mm_add_epi16(stp1[17], stp1[22]);
- stp2_18 = _mm_add_epi16(stp1[18], stp1[21]);
- stp2_19 = _mm_add_epi16(stp1[19], stp1[20]);
- stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]);
- stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]);
- stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]);
- stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]);
-
- stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]);
- stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]);
- stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]);
- stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]);
- stp2_28 = _mm_add_epi16(stp1[27], stp1[28]);
- stp2_29 = _mm_add_epi16(stp1[26], stp1[29]);
- stp2_30 = _mm_add_epi16(stp1[25], stp1[30]);
- stp2_31 = _mm_add_epi16(stp1[24], stp1[31]);
- }
-
- /* Stage7 */
- {-#if AVOID_OVERFLOW
- const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27);
- const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27);
- const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26);
- const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26);
-
- const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25);
- const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25);
- const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24);
- const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24);
-#endif
- stp1[0] = _mm_add_epi16(stp2_0, stp2_15);
- stp1[1] = _mm_add_epi16(stp2_1, stp2_14);
- stp1[2] = _mm_add_epi16(stp2_2, stp2_13);
- stp1[3] = _mm_add_epi16(stp2_3, stp2_12);
- stp1[4] = _mm_add_epi16(stp2_4, stp2_11);
- stp1[5] = _mm_add_epi16(stp2_5, stp2_10);
- stp1[6] = _mm_add_epi16(stp2_6, stp2_9);
- stp1[7] = _mm_add_epi16(stp2_7, stp2_8);
- stp1[8] = _mm_sub_epi16(stp2_7, stp2_8);
- stp1[9] = _mm_sub_epi16(stp2_6, stp2_9);
- stp1[10] = _mm_sub_epi16(stp2_5, stp2_10);
- stp1[11] = _mm_sub_epi16(stp2_4, stp2_11);
- stp1[12] = _mm_sub_epi16(stp2_3, stp2_12);
- stp1[13] = _mm_sub_epi16(stp2_2, stp2_13);
- stp1[14] = _mm_sub_epi16(stp2_1, stp2_14);
- stp1[15] = _mm_sub_epi16(stp2_0, stp2_15);
-
- stp1[16] = stp2_16;
- stp1[17] = stp2_17;
- stp1[18] = stp2_18;
- stp1[19] = stp2_19;
-
-#if AVOID_OVERFLOW
- MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0,
- stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21],
- stp1[26])
- MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0,
- stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23],
- stp1[24])
-#else
- tmp0 = _mm_add_epi16(stp2_20, stp2_27);
- tmp1 = _mm_sub_epi16(stp2_27, stp2_20);
- tmp2 = _mm_add_epi16(stp2_21, stp2_26);
- tmp3 = _mm_sub_epi16(stp2_26, stp2_21);
-
- stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0);
- stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0);
- stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0);
- stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0);
-
- tmp0 = _mm_add_epi16(stp2_22, stp2_25);
- tmp1 = _mm_sub_epi16(stp2_25, stp2_22);
- tmp2 = _mm_add_epi16(stp2_23, stp2_24);
- tmp3 = _mm_sub_epi16(stp2_24, stp2_23);
-
- stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0);
- stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0);
- stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0);
- stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0);
-#endif
-
- stp1[28] = stp2_28;
- stp1[29] = stp2_29;
- stp1[30] = stp2_30;
- stp1[31] = stp2_31;
- }
-#undef AVOID_OVERFLOW
+ butterfly(v20, v27, stg6_0, stg4_0, &stp1[20], &stp1[27]);
+ butterfly(v21, v26, stg6_0, stg4_0, &stp1[21], &stp1[26]);
+ butterfly(v22, v25, stg6_0, stg4_0, &stp1[22], &stp1[25]);
+ butterfly(v23, v24, stg6_0, stg4_0, &stp1[23], &stp1[24]);
}
// Only upper-left 8x8 has non-zero coeff
@@ -722,81 +645,20 @@
in[7] = load_input_data(input + 224);
array_transpose_8x8(in, in);
- idct32_34(in, stp1);
+ idct32_34_first_half(in, stp1);
+ idct32_34_second_half(in, stp1);
// 1_D: Store 32 intermediate results for each 8x32 block.
- col[0] = _mm_add_epi16(stp1[0], stp1[31]);
- col[1] = _mm_add_epi16(stp1[1], stp1[30]);
- col[2] = _mm_add_epi16(stp1[2], stp1[29]);
- col[3] = _mm_add_epi16(stp1[3], stp1[28]);
- col[4] = _mm_add_epi16(stp1[4], stp1[27]);
- col[5] = _mm_add_epi16(stp1[5], stp1[26]);
- col[6] = _mm_add_epi16(stp1[6], stp1[25]);
- col[7] = _mm_add_epi16(stp1[7], stp1[24]);
- col[8] = _mm_add_epi16(stp1[8], stp1[23]);
- col[9] = _mm_add_epi16(stp1[9], stp1[22]);
- col[10] = _mm_add_epi16(stp1[10], stp1[21]);
- col[11] = _mm_add_epi16(stp1[11], stp1[20]);
- col[12] = _mm_add_epi16(stp1[12], stp1[19]);
- col[13] = _mm_add_epi16(stp1[13], stp1[18]);
- col[14] = _mm_add_epi16(stp1[14], stp1[17]);
- col[15] = _mm_add_epi16(stp1[15], stp1[16]);
- col[16] = _mm_sub_epi16(stp1[15], stp1[16]);
- col[17] = _mm_sub_epi16(stp1[14], stp1[17]);
- col[18] = _mm_sub_epi16(stp1[13], stp1[18]);
- col[19] = _mm_sub_epi16(stp1[12], stp1[19]);
- col[20] = _mm_sub_epi16(stp1[11], stp1[20]);
- col[21] = _mm_sub_epi16(stp1[10], stp1[21]);
- col[22] = _mm_sub_epi16(stp1[9], stp1[22]);
- col[23] = _mm_sub_epi16(stp1[8], stp1[23]);
- col[24] = _mm_sub_epi16(stp1[7], stp1[24]);
- col[25] = _mm_sub_epi16(stp1[6], stp1[25]);
- col[26] = _mm_sub_epi16(stp1[5], stp1[26]);
- col[27] = _mm_sub_epi16(stp1[4], stp1[27]);
- col[28] = _mm_sub_epi16(stp1[3], stp1[28]);
- col[29] = _mm_sub_epi16(stp1[2], stp1[29]);
- col[30] = _mm_sub_epi16(stp1[1], stp1[30]);
- col[31] = _mm_sub_epi16(stp1[0], stp1[31]);
+ simple_group_butterfly(stp1, col);
for (i = 0; i < 4; i++) {int j;
// Transpose 32x8 block to 8x32 block
array_transpose_8x8(col + i * 8, in);
- idct32_34(in, stp1);
+ idct32_34_first_half(in, stp1);
+ idct32_34_second_half(in, stp1);
// 2_D: Calculate the results and store them to destination.
- in[0] = _mm_add_epi16(stp1[0], stp1[31]);
- in[1] = _mm_add_epi16(stp1[1], stp1[30]);
- in[2] = _mm_add_epi16(stp1[2], stp1[29]);
- in[3] = _mm_add_epi16(stp1[3], stp1[28]);
- in[4] = _mm_add_epi16(stp1[4], stp1[27]);
- in[5] = _mm_add_epi16(stp1[5], stp1[26]);
- in[6] = _mm_add_epi16(stp1[6], stp1[25]);
- in[7] = _mm_add_epi16(stp1[7], stp1[24]);
- in[8] = _mm_add_epi16(stp1[8], stp1[23]);
- in[9] = _mm_add_epi16(stp1[9], stp1[22]);
- in[10] = _mm_add_epi16(stp1[10], stp1[21]);
- in[11] = _mm_add_epi16(stp1[11], stp1[20]);
- in[12] = _mm_add_epi16(stp1[12], stp1[19]);
- in[13] = _mm_add_epi16(stp1[13], stp1[18]);
- in[14] = _mm_add_epi16(stp1[14], stp1[17]);
- in[15] = _mm_add_epi16(stp1[15], stp1[16]);
- in[16] = _mm_sub_epi16(stp1[15], stp1[16]);
- in[17] = _mm_sub_epi16(stp1[14], stp1[17]);
- in[18] = _mm_sub_epi16(stp1[13], stp1[18]);
- in[19] = _mm_sub_epi16(stp1[12], stp1[19]);
- in[20] = _mm_sub_epi16(stp1[11], stp1[20]);
- in[21] = _mm_sub_epi16(stp1[10], stp1[21]);
- in[22] = _mm_sub_epi16(stp1[9], stp1[22]);
- in[23] = _mm_sub_epi16(stp1[8], stp1[23]);
- in[24] = _mm_sub_epi16(stp1[7], stp1[24]);
- in[25] = _mm_sub_epi16(stp1[6], stp1[25]);
- in[26] = _mm_sub_epi16(stp1[5], stp1[26]);
- in[27] = _mm_sub_epi16(stp1[4], stp1[27]);
- in[28] = _mm_sub_epi16(stp1[3], stp1[28]);
- in[29] = _mm_sub_epi16(stp1[2], stp1[29]);
- in[30] = _mm_sub_epi16(stp1[1], stp1[30]);
- in[31] = _mm_sub_epi16(stp1[0], stp1[31]);
-
+ simple_group_butterfly(stp1, in);
for (j = 0; j < 32; ++j) {// Final rounding and shift
in[j] = _mm_adds_epi16(in[j], final_rounding);