ref: 4684d286defc9757e10be3fc5102187f531bb98c
dir: /vpx_dsp/x86/inv_txfm_ssse3.c/
/* * Copyright (c) 2017 The WebM project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include <tmmintrin.h> #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/x86/inv_txfm_sse2.h" #include "vpx_dsp/x86/txfm_common_sse2.h" void vpx_idct8x8_64_add_ssse3(const tran_low_t *input, uint8_t *dest, int stride) { const __m128i zero = _mm_setzero_si128(); const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); const __m128i final_rounding = _mm_set1_epi16(1 << 4); const __m128i stg1_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); const __m128i stg1_1 = pair_set_epi16(cospi_4_64, cospi_28_64); const __m128i stg1_2 = pair_set_epi16(-cospi_20_64, cospi_12_64); const __m128i stg1_3 = pair_set_epi16(cospi_12_64, cospi_20_64); const __m128i stk2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); const __m128i stk2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stg2_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); const __m128i stg2_3 = pair_set_epi16(cospi_8_64, cospi_24_64); __m128i in0, in1, in2, in3, in4, in5, in6, in7; __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; int i; // Load input data. in0 = load_input_data(input); in1 = load_input_data(input + 8 * 1); in2 = load_input_data(input + 8 * 2); in3 = load_input_data(input + 8 * 3); in4 = load_input_data(input + 8 * 4); in5 = load_input_data(input + 8 * 5); in6 = load_input_data(input + 8 * 6); in7 = load_input_data(input + 8 * 7); // 2-D for (i = 0; i < 2; i++) { // 8x8 Transpose is copied from vpx_fdct8x8_sse2() TRANSPOSE_8X8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, in5, in6, in7); // 4-stage 1D idct8x8 { /* Stage1 */ { const __m128i lo_17 = _mm_unpacklo_epi16(in1, in7); const __m128i hi_17 = _mm_unpackhi_epi16(in1, in7); const __m128i lo_35 = _mm_unpacklo_epi16(in3, in5); const __m128i hi_35 = _mm_unpackhi_epi16(in3, in5); { tmp0 = _mm_madd_epi16(lo_17, stg1_0); tmp1 = _mm_madd_epi16(hi_17, stg1_0); tmp2 = _mm_madd_epi16(lo_17, stg1_1); tmp3 = _mm_madd_epi16(hi_17, stg1_1); tmp4 = _mm_madd_epi16(lo_35, stg1_2); tmp5 = _mm_madd_epi16(hi_35, stg1_2); tmp6 = _mm_madd_epi16(lo_35, stg1_3); tmp7 = _mm_madd_epi16(hi_35, stg1_3); tmp0 = _mm_add_epi32(tmp0, rounding); tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp4 = _mm_add_epi32(tmp4, rounding); tmp5 = _mm_add_epi32(tmp5, rounding); tmp6 = _mm_add_epi32(tmp6, rounding); tmp7 = _mm_add_epi32(tmp7, rounding); tmp0 = _mm_srai_epi32(tmp0, 14); tmp1 = _mm_srai_epi32(tmp1, 14); tmp2 = _mm_srai_epi32(tmp2, 14); tmp3 = _mm_srai_epi32(tmp3, 14); tmp4 = _mm_srai_epi32(tmp4, 14); tmp5 = _mm_srai_epi32(tmp5, 14); tmp6 = _mm_srai_epi32(tmp6, 14); tmp7 = _mm_srai_epi32(tmp7, 14); stp1_4 = _mm_packs_epi32(tmp0, tmp1); stp1_7 = _mm_packs_epi32(tmp2, tmp3); stp1_5 = _mm_packs_epi32(tmp4, tmp5); stp1_6 = _mm_packs_epi32(tmp6, tmp7); } } /* Stage2 */ { const __m128i lo_26 = _mm_unpacklo_epi16(in2, in6); const __m128i hi_26 = _mm_unpackhi_epi16(in2, in6); { tmp0 = _mm_unpacklo_epi16(in0, in4); tmp1 = _mm_unpackhi_epi16(in0, in4); tmp2 = _mm_madd_epi16(tmp0, stk2_0); tmp3 = _mm_madd_epi16(tmp1, stk2_0); tmp4 = _mm_madd_epi16(tmp0, stk2_1); tmp5 = _mm_madd_epi16(tmp1, stk2_1); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp4 = _mm_add_epi32(tmp4, rounding); tmp5 = _mm_add_epi32(tmp5, rounding); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); stp2_0 = _mm_packs_epi32(tmp2, tmp3); stp2_1 = _mm_packs_epi32(tmp4, tmp5); tmp0 = _mm_madd_epi16(lo_26, stg2_2); tmp1 = _mm_madd_epi16(hi_26, stg2_2); tmp2 = _mm_madd_epi16(lo_26, stg2_3); tmp3 = _mm_madd_epi16(hi_26, stg2_3); tmp0 = _mm_add_epi32(tmp0, rounding); tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp0 = _mm_srai_epi32(tmp0, 14); tmp1 = _mm_srai_epi32(tmp1, 14); tmp2 = _mm_srai_epi32(tmp2, 14); tmp3 = _mm_srai_epi32(tmp3, 14); stp2_2 = _mm_packs_epi32(tmp0, tmp1); stp2_3 = _mm_packs_epi32(tmp2, tmp3); } stp2_4 = _mm_add_epi16(stp1_4, stp1_5); stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); stp2_7 = _mm_add_epi16(stp1_7, stp1_6); } /* Stage3 */ { stp1_0 = _mm_add_epi16(stp2_0, stp2_3); stp1_1 = _mm_add_epi16(stp2_1, stp2_2); stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); tmp0 = _mm_unpacklo_epi16(stp2_6, stp2_5); tmp1 = _mm_unpackhi_epi16(stp2_6, stp2_5); tmp2 = _mm_madd_epi16(tmp0, stk2_1); tmp3 = _mm_madd_epi16(tmp1, stk2_1); tmp4 = _mm_madd_epi16(tmp0, stk2_0); tmp5 = _mm_madd_epi16(tmp1, stk2_0); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp4 = _mm_add_epi32(tmp4, rounding); tmp5 = _mm_add_epi32(tmp5, rounding); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); tmp4 = _mm_srai_epi32(tmp4, DCT_CONST_BITS); tmp5 = _mm_srai_epi32(tmp5, DCT_CONST_BITS); stp1_5 = _mm_packs_epi32(tmp2, tmp3); stp1_6 = _mm_packs_epi32(tmp4, tmp5); } /* Stage4 */ in0 = _mm_add_epi16(stp1_0, stp2_7); in1 = _mm_add_epi16(stp1_1, stp1_6); in2 = _mm_add_epi16(stp1_2, stp1_5); in3 = _mm_add_epi16(stp1_3, stp2_4); in4 = _mm_sub_epi16(stp1_3, stp2_4); in5 = _mm_sub_epi16(stp1_2, stp1_5); in6 = _mm_sub_epi16(stp1_1, stp1_6); in7 = _mm_sub_epi16(stp1_0, stp2_7); } } // Final rounding and shift in0 = _mm_adds_epi16(in0, final_rounding); in1 = _mm_adds_epi16(in1, final_rounding); in2 = _mm_adds_epi16(in2, final_rounding); in3 = _mm_adds_epi16(in3, final_rounding); in4 = _mm_adds_epi16(in4, final_rounding); in5 = _mm_adds_epi16(in5, final_rounding); in6 = _mm_adds_epi16(in6, final_rounding); in7 = _mm_adds_epi16(in7, final_rounding); in0 = _mm_srai_epi16(in0, 5); in1 = _mm_srai_epi16(in1, 5); in2 = _mm_srai_epi16(in2, 5); in3 = _mm_srai_epi16(in3, 5); in4 = _mm_srai_epi16(in4, 5); in5 = _mm_srai_epi16(in5, 5); in6 = _mm_srai_epi16(in6, 5); in7 = _mm_srai_epi16(in7, 5); RECON_AND_STORE(dest + 0 * stride, in0); RECON_AND_STORE(dest + 1 * stride, in1); RECON_AND_STORE(dest + 2 * stride, in2); RECON_AND_STORE(dest + 3 * stride, in3); RECON_AND_STORE(dest + 4 * stride, in4); RECON_AND_STORE(dest + 5 * stride, in5); RECON_AND_STORE(dest + 6 * stride, in6); RECON_AND_STORE(dest + 7 * stride, in7); } void vpx_idct8x8_12_add_ssse3(const tran_low_t *input, uint8_t *dest, int stride) { const __m128i zero = _mm_setzero_si128(); const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); const __m128i final_rounding = _mm_set1_epi16(1 << 4); const __m128i stg1_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); const __m128i stg1_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); const __m128i stg1_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64); const __m128i stg1_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64); const __m128i stg2_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); const __m128i stk2_0 = pair_set_epi16(cospi_16_64, cospi_16_64); const __m128i stk2_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stg2_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64); const __m128i stg2_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64); const __m128i stg3_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); __m128i in0, in1, in2, in3, in4, in5, in6, in7; __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7; __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7; __m128i tmp0, tmp1, tmp2, tmp3; // Rows. Load 4-row input data. in0 = load_input_data(input); in1 = load_input_data(input + 8 * 1); in2 = load_input_data(input + 8 * 2); in3 = load_input_data(input + 8 * 3); // 8x4 Transpose TRANSPOSE_8X8_10(in0, in1, in2, in3, in0, in1); // Stage1 tmp0 = _mm_mulhrs_epi16(in0, stg1_0); tmp1 = _mm_mulhrs_epi16(in0, stg1_1); tmp2 = _mm_mulhrs_epi16(in1, stg1_2); tmp3 = _mm_mulhrs_epi16(in1, stg1_3); stp1_4 = _mm_unpackhi_epi64(tmp0, tmp1); stp1_5 = _mm_unpackhi_epi64(tmp2, tmp3); // Stage2 tmp0 = _mm_mulhrs_epi16(in0, stg2_0); stp2_0 = _mm_unpacklo_epi64(tmp0, tmp0); tmp1 = _mm_mulhrs_epi16(in1, stg2_2); tmp2 = _mm_mulhrs_epi16(in1, stg2_3); stp2_2 = _mm_unpacklo_epi64(tmp2, tmp1); tmp0 = _mm_add_epi16(stp1_4, stp1_5); tmp1 = _mm_sub_epi16(stp1_4, stp1_5); stp2_4 = tmp0; stp2_5 = _mm_unpacklo_epi64(tmp1, zero); stp2_6 = _mm_unpackhi_epi64(tmp1, zero); tmp0 = _mm_unpacklo_epi16(stp2_5, stp2_6); tmp1 = _mm_madd_epi16(tmp0, stg3_0); tmp2 = _mm_madd_epi16(tmp0, stk2_0); // stg3_1 = stk2_0 tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); stp1_5 = _mm_packs_epi32(tmp1, tmp2); // Stage3 tmp2 = _mm_add_epi16(stp2_0, stp2_2); tmp3 = _mm_sub_epi16(stp2_0, stp2_2); stp1_2 = _mm_unpackhi_epi64(tmp3, tmp2); stp1_3 = _mm_unpacklo_epi64(tmp3, tmp2); // Stage4 tmp0 = _mm_add_epi16(stp1_3, stp2_4); tmp1 = _mm_add_epi16(stp1_2, stp1_5); tmp2 = _mm_sub_epi16(stp1_3, stp2_4); tmp3 = _mm_sub_epi16(stp1_2, stp1_5); TRANSPOSE_4X8_10(tmp0, tmp1, tmp2, tmp3, in0, in1, in2, in3) /* Stage1 */ stp1_4 = _mm_mulhrs_epi16(in1, stg1_0); stp1_7 = _mm_mulhrs_epi16(in1, stg1_1); stp1_5 = _mm_mulhrs_epi16(in3, stg1_2); stp1_6 = _mm_mulhrs_epi16(in3, stg1_3); /* Stage2 */ stp2_0 = _mm_mulhrs_epi16(in0, stg2_0); stp2_1 = _mm_mulhrs_epi16(in0, stg2_0); stp2_2 = _mm_mulhrs_epi16(in2, stg2_2); stp2_3 = _mm_mulhrs_epi16(in2, stg2_3); stp2_4 = _mm_add_epi16(stp1_4, stp1_5); stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); stp2_7 = _mm_add_epi16(stp1_7, stp1_6); /* Stage3 */ stp1_0 = _mm_add_epi16(stp2_0, stp2_3); stp1_1 = _mm_add_epi16(stp2_1, stp2_2); stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); tmp0 = _mm_unpacklo_epi16(stp2_6, stp2_5); tmp1 = _mm_unpackhi_epi16(stp2_6, stp2_5); tmp2 = _mm_madd_epi16(tmp0, stk2_0); tmp3 = _mm_madd_epi16(tmp1, stk2_0); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); stp1_6 = _mm_packs_epi32(tmp2, tmp3); tmp2 = _mm_madd_epi16(tmp0, stk2_1); tmp3 = _mm_madd_epi16(tmp1, stk2_1); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); stp1_5 = _mm_packs_epi32(tmp2, tmp3); /* Stage4 */ in0 = _mm_add_epi16(stp1_0, stp2_7); in1 = _mm_add_epi16(stp1_1, stp1_6); in2 = _mm_add_epi16(stp1_2, stp1_5); in3 = _mm_add_epi16(stp1_3, stp2_4); in4 = _mm_sub_epi16(stp1_3, stp2_4); in5 = _mm_sub_epi16(stp1_2, stp1_5); in6 = _mm_sub_epi16(stp1_1, stp1_6); in7 = _mm_sub_epi16(stp1_0, stp2_7); // Final rounding and shift in0 = _mm_adds_epi16(in0, final_rounding); in1 = _mm_adds_epi16(in1, final_rounding); in2 = _mm_adds_epi16(in2, final_rounding); in3 = _mm_adds_epi16(in3, final_rounding); in4 = _mm_adds_epi16(in4, final_rounding); in5 = _mm_adds_epi16(in5, final_rounding); in6 = _mm_adds_epi16(in6, final_rounding); in7 = _mm_adds_epi16(in7, final_rounding); in0 = _mm_srai_epi16(in0, 5); in1 = _mm_srai_epi16(in1, 5); in2 = _mm_srai_epi16(in2, 5); in3 = _mm_srai_epi16(in3, 5); in4 = _mm_srai_epi16(in4, 5); in5 = _mm_srai_epi16(in5, 5); in6 = _mm_srai_epi16(in6, 5); in7 = _mm_srai_epi16(in7, 5); RECON_AND_STORE(dest + 0 * stride, in0); RECON_AND_STORE(dest + 1 * stride, in1); RECON_AND_STORE(dest + 2 * stride, in2); RECON_AND_STORE(dest + 3 * stride, in3); RECON_AND_STORE(dest + 4 * stride, in4); RECON_AND_STORE(dest + 5 * stride, in5); RECON_AND_STORE(dest + 6 * stride, in6); RECON_AND_STORE(dest + 7 * stride, in7); } static INLINE void idct32_34(const __m128i *in, __m128i *stp1) { const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); // idct constants for each stage const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64); const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64); const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64); const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64); const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; /* Stage1 */ stp1[16] = _mm_mulhrs_epi16(in[1], stk1_0); stp1[31] = _mm_mulhrs_epi16(in[1], stk1_1); stp1[19] = _mm_mulhrs_epi16(in[7], stk1_6); stp1[28] = _mm_mulhrs_epi16(in[7], stk1_7); stp1[20] = _mm_mulhrs_epi16(in[5], stk1_8); stp1[27] = _mm_mulhrs_epi16(in[5], stk1_9); stp1[23] = _mm_mulhrs_epi16(in[3], stk1_14); stp1[24] = _mm_mulhrs_epi16(in[3], stk1_15); /* Stage2 */ stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0); stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1); stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6); stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7); /* Stage3 */ { const __m128i lo_17_30 = _mm_unpacklo_epi16(stp1[16], stp1[31]); const __m128i hi_17_30 = _mm_unpackhi_epi16(stp1[16], stp1[31]); const __m128i lo_18_29 = _mm_unpacklo_epi16(stp1[19], stp1[28]); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp1[19], stp1[28]); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp1[20], stp1[27]); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp1[20], stp1[27]); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp1[23], stp1[24]); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp1[23], stp1[24]); stp1[4] = _mm_mulhrs_epi16(in[4], stk3_0); stp1[7] = _mm_mulhrs_epi16(in[4], stk3_1); MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, stg3_5, stg3_6, stg3_4, stp1[17], stp1[30], stp1[18], stp1[29]) MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, stg3_9, stg3_10, stg3_8, stp1[21], stp1[26], stp1[22], stp1[25]) } /* Stage4 */ { const __m128i lo_9_14 = _mm_unpacklo_epi16(stp2_8, stp2_15); const __m128i hi_9_14 = _mm_unpackhi_epi16(stp2_8, stp2_15); const __m128i lo_10_13 = _mm_unpacklo_epi16(stp2_11, stp2_12); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp2_11, stp2_12); stp1[0] = _mm_mulhrs_epi16(in[0], stk4_0); stp1[1] = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0 stp1[2] = stp1[0]; stp1[3] = stp1[1]; MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5, stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13) stp2_16 = _mm_add_epi16(stp1[16], stp1[19]); stp2_17 = _mm_add_epi16(stp1[17], stp1[18]); stp2_18 = _mm_sub_epi16(stp1[17], stp1[18]); stp2_19 = _mm_sub_epi16(stp1[16], stp1[19]); stp2_20 = _mm_sub_epi16(stp1[23], stp1[20]); stp2_21 = _mm_sub_epi16(stp1[22], stp1[21]); stp2_22 = _mm_add_epi16(stp1[22], stp1[21]); stp2_23 = _mm_add_epi16(stp1[23], stp1[20]); stp2_24 = _mm_add_epi16(stp1[24], stp1[27]); stp2_25 = _mm_add_epi16(stp1[25], stp1[26]); stp2_26 = _mm_sub_epi16(stp1[25], stp1[26]); stp2_27 = _mm_sub_epi16(stp1[24], stp1[27]); stp2_28 = _mm_sub_epi16(stp1[31], stp1[28]); stp2_29 = _mm_sub_epi16(stp1[30], stp1[29]); stp2_30 = _mm_add_epi16(stp1[29], stp1[30]); stp2_31 = _mm_add_epi16(stp1[28], stp1[31]); } /* Stage5 */ { // Note: // #define AVOID_OVERFLOW = 0, code would be faster. But it can't pass // SingleExtreme test. The MaxSupportedCoeff/MinSupportedCoeff must drop // to 23198 and -23197, respectively. #define AVOID_OVERFLOW (1) #if AVOID_OVERFLOW const __m128i lo_6_5 = _mm_unpacklo_epi16(stp1[7], stp1[4]); const __m128i hi_6_5 = _mm_unpackhi_epi16(stp1[7], stp1[4]); #endif const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); #if AVOID_OVERFLOW tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); tmp0 = _mm_add_epi32(tmp0, rounding); tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); stp1[5] = _mm_packs_epi32(tmp0, tmp1); stp1[6] = _mm_packs_epi32(tmp2, tmp3); #else tmp0 = _mm_sub_epi16(stp1[7], stp1[4]); tmp1 = _mm_adds_epi16(stp1[7], stp1[4]); stp1[5] = _mm_mulhrs_epi16(tmp0, stk4_0); stp1[6] = _mm_mulhrs_epi16(tmp1, stk4_0); #endif stp1[8] = _mm_add_epi16(stp2_8, stp2_11); stp1[9] = _mm_add_epi16(stp2_9, stp2_10); stp1[10] = _mm_sub_epi16(stp2_9, stp2_10); stp1[11] = _mm_sub_epi16(stp2_8, stp2_11); stp1[12] = _mm_sub_epi16(stp2_15, stp2_12); stp1[13] = _mm_sub_epi16(stp2_14, stp2_13); stp1[14] = _mm_add_epi16(stp2_14, stp2_13); stp1[15] = _mm_add_epi16(stp2_15, stp2_12); MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, stg4_5, stg4_4, stg4_5, stp1[18], stp1[29], stp1[19], stp1[28]) MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, stg4_4, stg4_6, stg4_4, stp1[20], stp1[27], stp1[21], stp1[26]) stp1[16] = stp2_16; stp1[17] = stp2_17; stp1[22] = stp2_22; stp1[23] = stp2_23; stp1[24] = stp2_24; stp1[25] = stp2_25; stp1[30] = stp2_30; stp1[31] = stp2_31; } /* Stage6 */ { #if AVOID_OVERFLOW const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1[10], stp1[13]); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1[10], stp1[13]); const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1[11], stp1[12]); const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1[11], stp1[12]); #endif stp2_0 = _mm_add_epi16(stp1[0], stp1[7]); stp2_1 = _mm_add_epi16(stp1[1], stp1[6]); stp2_2 = _mm_add_epi16(stp1[2], stp1[5]); stp2_3 = _mm_add_epi16(stp1[3], stp1[4]); stp2_4 = _mm_sub_epi16(stp1[3], stp1[4]); stp2_5 = _mm_sub_epi16(stp1[2], stp1[5]); stp2_6 = _mm_sub_epi16(stp1[1], stp1[6]); stp2_7 = _mm_sub_epi16(stp1[0], stp1[7]); stp2_8 = stp1[8]; stp2_9 = stp1[9]; stp2_14 = stp1[14]; stp2_15 = stp1[15]; #if AVOID_OVERFLOW MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, stp2_12) #else tmp0 = _mm_add_epi16(stp1[10], stp1[13]); tmp1 = _mm_sub_epi16(stp1[13], stp1[10]); tmp2 = _mm_add_epi16(stp1[11], stp1[12]); tmp3 = _mm_sub_epi16(stp1[12], stp1[11]); stp2_10 = _mm_mulhrs_epi16(tmp1, stk4_0); stp2_13 = _mm_mulhrs_epi16(tmp0, stk4_0); stp2_11 = _mm_mulhrs_epi16(tmp3, stk4_0); stp2_12 = _mm_mulhrs_epi16(tmp2, stk4_0); #endif stp2_16 = _mm_add_epi16(stp1[16], stp1[23]); stp2_17 = _mm_add_epi16(stp1[17], stp1[22]); stp2_18 = _mm_add_epi16(stp1[18], stp1[21]); stp2_19 = _mm_add_epi16(stp1[19], stp1[20]); stp2_20 = _mm_sub_epi16(stp1[19], stp1[20]); stp2_21 = _mm_sub_epi16(stp1[18], stp1[21]); stp2_22 = _mm_sub_epi16(stp1[17], stp1[22]); stp2_23 = _mm_sub_epi16(stp1[16], stp1[23]); stp2_24 = _mm_sub_epi16(stp1[31], stp1[24]); stp2_25 = _mm_sub_epi16(stp1[30], stp1[25]); stp2_26 = _mm_sub_epi16(stp1[29], stp1[26]); stp2_27 = _mm_sub_epi16(stp1[28], stp1[27]); stp2_28 = _mm_add_epi16(stp1[27], stp1[28]); stp2_29 = _mm_add_epi16(stp1[26], stp1[29]); stp2_30 = _mm_add_epi16(stp1[25], stp1[30]); stp2_31 = _mm_add_epi16(stp1[24], stp1[31]); } /* Stage7 */ { #if AVOID_OVERFLOW const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); #endif stp1[0] = _mm_add_epi16(stp2_0, stp2_15); stp1[1] = _mm_add_epi16(stp2_1, stp2_14); stp1[2] = _mm_add_epi16(stp2_2, stp2_13); stp1[3] = _mm_add_epi16(stp2_3, stp2_12); stp1[4] = _mm_add_epi16(stp2_4, stp2_11); stp1[5] = _mm_add_epi16(stp2_5, stp2_10); stp1[6] = _mm_add_epi16(stp2_6, stp2_9); stp1[7] = _mm_add_epi16(stp2_7, stp2_8); stp1[8] = _mm_sub_epi16(stp2_7, stp2_8); stp1[9] = _mm_sub_epi16(stp2_6, stp2_9); stp1[10] = _mm_sub_epi16(stp2_5, stp2_10); stp1[11] = _mm_sub_epi16(stp2_4, stp2_11); stp1[12] = _mm_sub_epi16(stp2_3, stp2_12); stp1[13] = _mm_sub_epi16(stp2_2, stp2_13); stp1[14] = _mm_sub_epi16(stp2_1, stp2_14); stp1[15] = _mm_sub_epi16(stp2_0, stp2_15); stp1[16] = stp2_16; stp1[17] = stp2_17; stp1[18] = stp2_18; stp1[19] = stp2_19; #if AVOID_OVERFLOW MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, stg4_0, stg6_0, stg4_0, stp1[20], stp1[27], stp1[21], stp1[26]) MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, stg4_0, stg6_0, stg4_0, stp1[22], stp1[25], stp1[23], stp1[24]) #else tmp0 = _mm_add_epi16(stp2_20, stp2_27); tmp1 = _mm_sub_epi16(stp2_27, stp2_20); tmp2 = _mm_add_epi16(stp2_21, stp2_26); tmp3 = _mm_sub_epi16(stp2_26, stp2_21); stp1[20] = _mm_mulhrs_epi16(tmp1, stk4_0); stp1[27] = _mm_mulhrs_epi16(tmp0, stk4_0); stp1[21] = _mm_mulhrs_epi16(tmp3, stk4_0); stp1[26] = _mm_mulhrs_epi16(tmp2, stk4_0); tmp0 = _mm_add_epi16(stp2_22, stp2_25); tmp1 = _mm_sub_epi16(stp2_25, stp2_22); tmp2 = _mm_add_epi16(stp2_23, stp2_24); tmp3 = _mm_sub_epi16(stp2_24, stp2_23); stp1[22] = _mm_mulhrs_epi16(tmp1, stk4_0); stp1[25] = _mm_mulhrs_epi16(tmp0, stk4_0); stp1[23] = _mm_mulhrs_epi16(tmp3, stk4_0); stp1[24] = _mm_mulhrs_epi16(tmp2, stk4_0); #endif stp1[28] = stp2_28; stp1[29] = stp2_29; stp1[30] = stp2_30; stp1[31] = stp2_31; } #undef AVOID_OVERFLOW } // Only upper-left 8x8 has non-zero coeff void vpx_idct32x32_34_add_ssse3(const tran_low_t *input, uint8_t *dest, int stride) { const __m128i zero = _mm_setzero_si128(); const __m128i final_rounding = _mm_set1_epi16(1 << 5); __m128i in[32], col[32]; __m128i stp1[32]; int i; // Load input data. Only need to load the top left 8x8 block. in[0] = load_input_data(input); in[1] = load_input_data(input + 32); in[2] = load_input_data(input + 64); in[3] = load_input_data(input + 96); in[4] = load_input_data(input + 128); in[5] = load_input_data(input + 160); in[6] = load_input_data(input + 192); in[7] = load_input_data(input + 224); array_transpose_8x8(in, in); idct32_34(in, stp1); // 1_D: Store 32 intermediate results for each 8x32 block. col[0] = _mm_add_epi16(stp1[0], stp1[31]); col[1] = _mm_add_epi16(stp1[1], stp1[30]); col[2] = _mm_add_epi16(stp1[2], stp1[29]); col[3] = _mm_add_epi16(stp1[3], stp1[28]); col[4] = _mm_add_epi16(stp1[4], stp1[27]); col[5] = _mm_add_epi16(stp1[5], stp1[26]); col[6] = _mm_add_epi16(stp1[6], stp1[25]); col[7] = _mm_add_epi16(stp1[7], stp1[24]); col[8] = _mm_add_epi16(stp1[8], stp1[23]); col[9] = _mm_add_epi16(stp1[9], stp1[22]); col[10] = _mm_add_epi16(stp1[10], stp1[21]); col[11] = _mm_add_epi16(stp1[11], stp1[20]); col[12] = _mm_add_epi16(stp1[12], stp1[19]); col[13] = _mm_add_epi16(stp1[13], stp1[18]); col[14] = _mm_add_epi16(stp1[14], stp1[17]); col[15] = _mm_add_epi16(stp1[15], stp1[16]); col[16] = _mm_sub_epi16(stp1[15], stp1[16]); col[17] = _mm_sub_epi16(stp1[14], stp1[17]); col[18] = _mm_sub_epi16(stp1[13], stp1[18]); col[19] = _mm_sub_epi16(stp1[12], stp1[19]); col[20] = _mm_sub_epi16(stp1[11], stp1[20]); col[21] = _mm_sub_epi16(stp1[10], stp1[21]); col[22] = _mm_sub_epi16(stp1[9], stp1[22]); col[23] = _mm_sub_epi16(stp1[8], stp1[23]); col[24] = _mm_sub_epi16(stp1[7], stp1[24]); col[25] = _mm_sub_epi16(stp1[6], stp1[25]); col[26] = _mm_sub_epi16(stp1[5], stp1[26]); col[27] = _mm_sub_epi16(stp1[4], stp1[27]); col[28] = _mm_sub_epi16(stp1[3], stp1[28]); col[29] = _mm_sub_epi16(stp1[2], stp1[29]); col[30] = _mm_sub_epi16(stp1[1], stp1[30]); col[31] = _mm_sub_epi16(stp1[0], stp1[31]); for (i = 0; i < 4; i++) { int j; // Transpose 32x8 block to 8x32 block array_transpose_8x8(col + i * 8, in); idct32_34(in, stp1); // 2_D: Calculate the results and store them to destination. in[0] = _mm_add_epi16(stp1[0], stp1[31]); in[1] = _mm_add_epi16(stp1[1], stp1[30]); in[2] = _mm_add_epi16(stp1[2], stp1[29]); in[3] = _mm_add_epi16(stp1[3], stp1[28]); in[4] = _mm_add_epi16(stp1[4], stp1[27]); in[5] = _mm_add_epi16(stp1[5], stp1[26]); in[6] = _mm_add_epi16(stp1[6], stp1[25]); in[7] = _mm_add_epi16(stp1[7], stp1[24]); in[8] = _mm_add_epi16(stp1[8], stp1[23]); in[9] = _mm_add_epi16(stp1[9], stp1[22]); in[10] = _mm_add_epi16(stp1[10], stp1[21]); in[11] = _mm_add_epi16(stp1[11], stp1[20]); in[12] = _mm_add_epi16(stp1[12], stp1[19]); in[13] = _mm_add_epi16(stp1[13], stp1[18]); in[14] = _mm_add_epi16(stp1[14], stp1[17]); in[15] = _mm_add_epi16(stp1[15], stp1[16]); in[16] = _mm_sub_epi16(stp1[15], stp1[16]); in[17] = _mm_sub_epi16(stp1[14], stp1[17]); in[18] = _mm_sub_epi16(stp1[13], stp1[18]); in[19] = _mm_sub_epi16(stp1[12], stp1[19]); in[20] = _mm_sub_epi16(stp1[11], stp1[20]); in[21] = _mm_sub_epi16(stp1[10], stp1[21]); in[22] = _mm_sub_epi16(stp1[9], stp1[22]); in[23] = _mm_sub_epi16(stp1[8], stp1[23]); in[24] = _mm_sub_epi16(stp1[7], stp1[24]); in[25] = _mm_sub_epi16(stp1[6], stp1[25]); in[26] = _mm_sub_epi16(stp1[5], stp1[26]); in[27] = _mm_sub_epi16(stp1[4], stp1[27]); in[28] = _mm_sub_epi16(stp1[3], stp1[28]); in[29] = _mm_sub_epi16(stp1[2], stp1[29]); in[30] = _mm_sub_epi16(stp1[1], stp1[30]); in[31] = _mm_sub_epi16(stp1[0], stp1[31]); for (j = 0; j < 32; ++j) { // Final rounding and shift in[j] = _mm_adds_epi16(in[j], final_rounding); in[j] = _mm_srai_epi16(in[j], 6); RECON_AND_STORE(dest + j * stride, in[j]); } dest += 8; } } // in0[16] represents the left 8x16 block // in1[16] represents the right 8x16 block static void load_buffer_16x16(const tran_low_t *input, __m128i *in0, __m128i *in1) { int i; for (i = 0; i < 16; i++) { in0[i] = load_input_data(input); in1[i] = load_input_data(input + 8); input += 32; } } static void array_transpose_16x16_2(__m128i *in0, __m128i *in1, __m128i *out0, __m128i *out1) { array_transpose_8x8(in0, out0); array_transpose_8x8(&in0[8], out1); array_transpose_8x8(in1, &out0[8]); array_transpose_8x8(&in1[8], &out1[8]); } // For each 8x16 block __m128i in[16], output __m128i col[32] static void idct32_8x16_135(__m128i *in) { const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); const __m128i stk1_0 = pair_set_epi16(2 * cospi_31_64, 2 * cospi_31_64); const __m128i stk1_1 = pair_set_epi16(2 * cospi_1_64, 2 * cospi_1_64); const __m128i stk1_2 = pair_set_epi16(-2 * cospi_17_64, -2 * cospi_17_64); const __m128i stk1_3 = pair_set_epi16(2 * cospi_15_64, 2 * cospi_15_64); const __m128i stk1_4 = pair_set_epi16(2 * cospi_23_64, 2 * cospi_23_64); const __m128i stk1_5 = pair_set_epi16(2 * cospi_9_64, 2 * cospi_9_64); const __m128i stk1_6 = pair_set_epi16(-2 * cospi_25_64, -2 * cospi_25_64); const __m128i stk1_7 = pair_set_epi16(2 * cospi_7_64, 2 * cospi_7_64); const __m128i stk1_8 = pair_set_epi16(2 * cospi_27_64, 2 * cospi_27_64); const __m128i stk1_9 = pair_set_epi16(2 * cospi_5_64, 2 * cospi_5_64); const __m128i stk1_10 = pair_set_epi16(-2 * cospi_21_64, -2 * cospi_21_64); const __m128i stk1_11 = pair_set_epi16(2 * cospi_11_64, 2 * cospi_11_64); const __m128i stk1_12 = pair_set_epi16(2 * cospi_19_64, 2 * cospi_19_64); const __m128i stk1_13 = pair_set_epi16(2 * cospi_13_64, 2 * cospi_13_64); const __m128i stk1_14 = pair_set_epi16(-2 * cospi_29_64, -2 * cospi_29_64); const __m128i stk1_15 = pair_set_epi16(2 * cospi_3_64, 2 * cospi_3_64); const __m128i stk2_0 = pair_set_epi16(2 * cospi_30_64, 2 * cospi_30_64); const __m128i stk2_1 = pair_set_epi16(2 * cospi_2_64, 2 * cospi_2_64); const __m128i stk2_2 = pair_set_epi16(-2 * cospi_18_64, -2 * cospi_18_64); const __m128i stk2_3 = pair_set_epi16(2 * cospi_14_64, 2 * cospi_14_64); const __m128i stk2_4 = pair_set_epi16(2 * cospi_22_64, 2 * cospi_22_64); const __m128i stk2_5 = pair_set_epi16(2 * cospi_10_64, 2 * cospi_10_64); const __m128i stk2_6 = pair_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64); const __m128i stk2_7 = pair_set_epi16(2 * cospi_6_64, 2 * cospi_6_64); const __m128i stk3_0 = pair_set_epi16(2 * cospi_28_64, 2 * cospi_28_64); const __m128i stk3_1 = pair_set_epi16(2 * cospi_4_64, 2 * cospi_4_64); const __m128i stk3_2 = pair_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64); const __m128i stk3_3 = pair_set_epi16(2 * cospi_12_64, 2 * cospi_12_64); const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stk4_0 = pair_set_epi16(2 * cospi_16_64, 2 * cospi_16_64); const __m128i stk4_2 = pair_set_epi16(2 * cospi_24_64, 2 * cospi_24_64); const __m128i stk4_3 = pair_set_epi16(2 * cospi_8_64, 2 * cospi_8_64); const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; /* Stage1 */ stp1_16 = _mm_mulhrs_epi16(in[1], stk1_0); stp1_31 = _mm_mulhrs_epi16(in[1], stk1_1); stp1_17 = _mm_mulhrs_epi16(in[15], stk1_2); stp1_30 = _mm_mulhrs_epi16(in[15], stk1_3); stp1_18 = _mm_mulhrs_epi16(in[9], stk1_4); stp1_29 = _mm_mulhrs_epi16(in[9], stk1_5); stp1_19 = _mm_mulhrs_epi16(in[7], stk1_6); stp1_28 = _mm_mulhrs_epi16(in[7], stk1_7); stp1_20 = _mm_mulhrs_epi16(in[5], stk1_8); stp1_27 = _mm_mulhrs_epi16(in[5], stk1_9); stp1_21 = _mm_mulhrs_epi16(in[11], stk1_10); stp1_26 = _mm_mulhrs_epi16(in[11], stk1_11); stp1_22 = _mm_mulhrs_epi16(in[13], stk1_12); stp1_25 = _mm_mulhrs_epi16(in[13], stk1_13); stp1_23 = _mm_mulhrs_epi16(in[3], stk1_14); stp1_24 = _mm_mulhrs_epi16(in[3], stk1_15); /* Stage2 */ stp2_8 = _mm_mulhrs_epi16(in[2], stk2_0); stp2_15 = _mm_mulhrs_epi16(in[2], stk2_1); stp2_9 = _mm_mulhrs_epi16(in[14], stk2_2); stp2_14 = _mm_mulhrs_epi16(in[14], stk2_3); stp2_10 = _mm_mulhrs_epi16(in[10], stk2_4); stp2_13 = _mm_mulhrs_epi16(in[10], stk2_5); stp2_11 = _mm_mulhrs_epi16(in[6], stk2_6); stp2_12 = _mm_mulhrs_epi16(in[6], stk2_7); stp2_16 = _mm_add_epi16(stp1_16, stp1_17); stp2_17 = _mm_sub_epi16(stp1_16, stp1_17); stp2_18 = _mm_sub_epi16(stp1_19, stp1_18); stp2_19 = _mm_add_epi16(stp1_19, stp1_18); stp2_20 = _mm_add_epi16(stp1_20, stp1_21); stp2_21 = _mm_sub_epi16(stp1_20, stp1_21); stp2_22 = _mm_sub_epi16(stp1_23, stp1_22); stp2_23 = _mm_add_epi16(stp1_23, stp1_22); stp2_24 = _mm_add_epi16(stp1_24, stp1_25); stp2_25 = _mm_sub_epi16(stp1_24, stp1_25); stp2_26 = _mm_sub_epi16(stp1_27, stp1_26); stp2_27 = _mm_add_epi16(stp1_27, stp1_26); stp2_28 = _mm_add_epi16(stp1_28, stp1_29); stp2_29 = _mm_sub_epi16(stp1_28, stp1_29); stp2_30 = _mm_sub_epi16(stp1_31, stp1_30); stp2_31 = _mm_add_epi16(stp1_31, stp1_30); /* Stage3 */ { const __m128i lo_17_30 = _mm_unpacklo_epi16(stp2_17, stp2_30); const __m128i hi_17_30 = _mm_unpackhi_epi16(stp2_17, stp2_30); const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); stp1_4 = _mm_mulhrs_epi16(in[4], stk3_0); stp1_7 = _mm_mulhrs_epi16(in[4], stk3_1); stp1_5 = _mm_mulhrs_epi16(in[12], stk3_2); stp1_6 = _mm_mulhrs_epi16(in[12], stk3_3); stp2_0 = _mm_mulhrs_epi16(in[0], stk4_0); stp2_1 = _mm_mulhrs_epi16(in[0], stk4_0); // stk4_1 = stk4_0 stp2_2 = _mm_mulhrs_epi16(in[8], stk4_2); stp2_3 = _mm_mulhrs_epi16(in[8], stk4_3); stp1_8 = _mm_add_epi16(stp2_8, stp2_9); stp1_9 = _mm_sub_epi16(stp2_8, stp2_9); stp1_10 = _mm_sub_epi16(stp2_11, stp2_10); stp1_11 = _mm_add_epi16(stp2_11, stp2_10); stp1_12 = _mm_add_epi16(stp2_12, stp2_13); stp1_13 = _mm_sub_epi16(stp2_12, stp2_13); stp1_14 = _mm_sub_epi16(stp2_15, stp2_14); stp1_15 = _mm_add_epi16(stp2_15, stp2_14); MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18, stp1_29) MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22, stp1_25) stp1_16 = stp2_16; stp1_31 = stp2_31; stp1_19 = stp2_19; stp1_20 = stp2_20; stp1_23 = stp2_23; stp1_24 = stp2_24; stp1_27 = stp2_27; stp1_28 = stp2_28; } /* Stage4 */ { const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); stp2_4 = _mm_add_epi16(stp1_4, stp1_5); stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); stp2_7 = _mm_add_epi16(stp1_7, stp1_6); MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5, stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13) stp2_8 = stp1_8; stp2_15 = stp1_15; stp2_11 = stp1_11; stp2_12 = stp1_12; stp2_16 = _mm_add_epi16(stp1_16, stp1_19); stp2_17 = _mm_add_epi16(stp1_17, stp1_18); stp2_18 = _mm_sub_epi16(stp1_17, stp1_18); stp2_19 = _mm_sub_epi16(stp1_16, stp1_19); stp2_20 = _mm_sub_epi16(stp1_23, stp1_20); stp2_21 = _mm_sub_epi16(stp1_22, stp1_21); stp2_22 = _mm_add_epi16(stp1_22, stp1_21); stp2_23 = _mm_add_epi16(stp1_23, stp1_20); stp2_24 = _mm_add_epi16(stp1_24, stp1_27); stp2_25 = _mm_add_epi16(stp1_25, stp1_26); stp2_26 = _mm_sub_epi16(stp1_25, stp1_26); stp2_27 = _mm_sub_epi16(stp1_24, stp1_27); stp2_28 = _mm_sub_epi16(stp1_31, stp1_28); stp2_29 = _mm_sub_epi16(stp1_30, stp1_29); stp2_30 = _mm_add_epi16(stp1_29, stp1_30); stp2_31 = _mm_add_epi16(stp1_28, stp1_31); } /* Stage5 */ { const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); stp1_0 = _mm_add_epi16(stp2_0, stp2_3); stp1_1 = _mm_add_epi16(stp2_1, stp2_2); stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); tmp0 = _mm_add_epi32(tmp0, rounding); tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); stp1_5 = _mm_packs_epi32(tmp0, tmp1); stp1_6 = _mm_packs_epi32(tmp2, tmp3); stp1_4 = stp2_4; stp1_7 = stp2_7; stp1_8 = _mm_add_epi16(stp2_8, stp2_11); stp1_9 = _mm_add_epi16(stp2_9, stp2_10); stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); stp1_11 = _mm_sub_epi16(stp2_8, stp2_11); stp1_12 = _mm_sub_epi16(stp2_15, stp2_12); stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); stp1_14 = _mm_add_epi16(stp2_14, stp2_13); stp1_15 = _mm_add_epi16(stp2_15, stp2_12); stp1_16 = stp2_16; stp1_17 = stp2_17; MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19, stp1_28) MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21, stp1_26) stp1_22 = stp2_22; stp1_23 = stp2_23; stp1_24 = stp2_24; stp1_25 = stp2_25; stp1_30 = stp2_30; stp1_31 = stp2_31; } /* Stage6 */ { const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); stp2_0 = _mm_add_epi16(stp1_0, stp1_7); stp2_1 = _mm_add_epi16(stp1_1, stp1_6); stp2_2 = _mm_add_epi16(stp1_2, stp1_5); stp2_3 = _mm_add_epi16(stp1_3, stp1_4); stp2_4 = _mm_sub_epi16(stp1_3, stp1_4); stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); stp2_7 = _mm_sub_epi16(stp1_0, stp1_7); stp2_8 = stp1_8; stp2_9 = stp1_9; stp2_14 = stp1_14; stp2_15 = stp1_15; MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, stp2_12) stp2_16 = _mm_add_epi16(stp1_16, stp1_23); stp2_17 = _mm_add_epi16(stp1_17, stp1_22); stp2_18 = _mm_add_epi16(stp1_18, stp1_21); stp2_19 = _mm_add_epi16(stp1_19, stp1_20); stp2_20 = _mm_sub_epi16(stp1_19, stp1_20); stp2_21 = _mm_sub_epi16(stp1_18, stp1_21); stp2_22 = _mm_sub_epi16(stp1_17, stp1_22); stp2_23 = _mm_sub_epi16(stp1_16, stp1_23); stp2_24 = _mm_sub_epi16(stp1_31, stp1_24); stp2_25 = _mm_sub_epi16(stp1_30, stp1_25); stp2_26 = _mm_sub_epi16(stp1_29, stp1_26); stp2_27 = _mm_sub_epi16(stp1_28, stp1_27); stp2_28 = _mm_add_epi16(stp1_27, stp1_28); stp2_29 = _mm_add_epi16(stp1_26, stp1_29); stp2_30 = _mm_add_epi16(stp1_25, stp1_30); stp2_31 = _mm_add_epi16(stp1_24, stp1_31); } /* Stage7 */ { const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); stp1_0 = _mm_add_epi16(stp2_0, stp2_15); stp1_1 = _mm_add_epi16(stp2_1, stp2_14); stp1_2 = _mm_add_epi16(stp2_2, stp2_13); stp1_3 = _mm_add_epi16(stp2_3, stp2_12); stp1_4 = _mm_add_epi16(stp2_4, stp2_11); stp1_5 = _mm_add_epi16(stp2_5, stp2_10); stp1_6 = _mm_add_epi16(stp2_6, stp2_9); stp1_7 = _mm_add_epi16(stp2_7, stp2_8); stp1_8 = _mm_sub_epi16(stp2_7, stp2_8); stp1_9 = _mm_sub_epi16(stp2_6, stp2_9); stp1_10 = _mm_sub_epi16(stp2_5, stp2_10); stp1_11 = _mm_sub_epi16(stp2_4, stp2_11); stp1_12 = _mm_sub_epi16(stp2_3, stp2_12); stp1_13 = _mm_sub_epi16(stp2_2, stp2_13); stp1_14 = _mm_sub_epi16(stp2_1, stp2_14); stp1_15 = _mm_sub_epi16(stp2_0, stp2_15); stp1_16 = stp2_16; stp1_17 = stp2_17; stp1_18 = stp2_18; stp1_19 = stp2_19; MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21, stp1_26) MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23, stp1_24) stp1_28 = stp2_28; stp1_29 = stp2_29; stp1_30 = stp2_30; stp1_31 = stp2_31; } in[0] = _mm_add_epi16(stp1_0, stp1_31); in[1] = _mm_add_epi16(stp1_1, stp1_30); in[2] = _mm_add_epi16(stp1_2, stp1_29); in[3] = _mm_add_epi16(stp1_3, stp1_28); in[4] = _mm_add_epi16(stp1_4, stp1_27); in[5] = _mm_add_epi16(stp1_5, stp1_26); in[6] = _mm_add_epi16(stp1_6, stp1_25); in[7] = _mm_add_epi16(stp1_7, stp1_24); in[8] = _mm_add_epi16(stp1_8, stp1_23); in[9] = _mm_add_epi16(stp1_9, stp1_22); in[10] = _mm_add_epi16(stp1_10, stp1_21); in[11] = _mm_add_epi16(stp1_11, stp1_20); in[12] = _mm_add_epi16(stp1_12, stp1_19); in[13] = _mm_add_epi16(stp1_13, stp1_18); in[14] = _mm_add_epi16(stp1_14, stp1_17); in[15] = _mm_add_epi16(stp1_15, stp1_16); in[16] = _mm_sub_epi16(stp1_15, stp1_16); in[17] = _mm_sub_epi16(stp1_14, stp1_17); in[18] = _mm_sub_epi16(stp1_13, stp1_18); in[19] = _mm_sub_epi16(stp1_12, stp1_19); in[20] = _mm_sub_epi16(stp1_11, stp1_20); in[21] = _mm_sub_epi16(stp1_10, stp1_21); in[22] = _mm_sub_epi16(stp1_9, stp1_22); in[23] = _mm_sub_epi16(stp1_8, stp1_23); in[24] = _mm_sub_epi16(stp1_7, stp1_24); in[25] = _mm_sub_epi16(stp1_6, stp1_25); in[26] = _mm_sub_epi16(stp1_5, stp1_26); in[27] = _mm_sub_epi16(stp1_4, stp1_27); in[28] = _mm_sub_epi16(stp1_3, stp1_28); in[29] = _mm_sub_epi16(stp1_2, stp1_29); in[30] = _mm_sub_epi16(stp1_1, stp1_30); in[31] = _mm_sub_epi16(stp1_0, stp1_31); } static INLINE void store_buffer_8x32(__m128i *in, uint8_t *dst, int stride) { const __m128i final_rounding = _mm_set1_epi16(1 << 5); const __m128i zero = _mm_setzero_si128(); int j = 0; while (j < 32) { in[j] = _mm_adds_epi16(in[j], final_rounding); in[j + 1] = _mm_adds_epi16(in[j + 1], final_rounding); in[j] = _mm_srai_epi16(in[j], 6); in[j + 1] = _mm_srai_epi16(in[j + 1], 6); RECON_AND_STORE(dst, in[j]); dst += stride; RECON_AND_STORE(dst, in[j + 1]); dst += stride; j += 2; } } static INLINE void recon_and_store(__m128i *in0, __m128i *in1, uint8_t *dest, int stride) { store_buffer_8x32(in0, dest, stride); store_buffer_8x32(in1, dest + 8, stride); } static INLINE void idct32_135(__m128i *col0, __m128i *col1) { idct32_8x16_135(col0); idct32_8x16_135(col1); } // Only upper-left 16x16 has non-zero coeff void vpx_idct32x32_135_add_ssse3(const tran_low_t *input, uint8_t *dest, int stride) { __m128i col0[32], col1[32], col2[32], col3[32]; // Load input data. Only need to load the top left 16x16 block. load_buffer_16x16(input, col2, col3); // columns array_transpose_16x16_2(col2, col3, col0, col1); idct32_135(col0, col1); // rows array_transpose_16x16_2(col0, col1, col2, col3); idct32_135(col2, col3); recon_and_store(col2, col3, dest, stride); array_transpose_16x16_2(&col0[16], &col1[16], col2, col3); idct32_135(col2, col3); recon_and_store(col2, col3, dest + 16, stride); } // For each 8x32 block __m128i in[32], output __m128i in[32] static void idct32_8x32(const __m128i *in, __m128i *out) { const __m128i rounding = _mm_set1_epi32(DCT_CONST_ROUNDING); // idct constants for each stage const __m128i stg1_0 = pair_set_epi16(cospi_31_64, -cospi_1_64); const __m128i stg1_1 = pair_set_epi16(cospi_1_64, cospi_31_64); const __m128i stg1_2 = pair_set_epi16(cospi_15_64, -cospi_17_64); const __m128i stg1_3 = pair_set_epi16(cospi_17_64, cospi_15_64); const __m128i stg1_4 = pair_set_epi16(cospi_23_64, -cospi_9_64); const __m128i stg1_5 = pair_set_epi16(cospi_9_64, cospi_23_64); const __m128i stg1_6 = pair_set_epi16(cospi_7_64, -cospi_25_64); const __m128i stg1_7 = pair_set_epi16(cospi_25_64, cospi_7_64); const __m128i stg1_8 = pair_set_epi16(cospi_27_64, -cospi_5_64); const __m128i stg1_9 = pair_set_epi16(cospi_5_64, cospi_27_64); const __m128i stg1_10 = pair_set_epi16(cospi_11_64, -cospi_21_64); const __m128i stg1_11 = pair_set_epi16(cospi_21_64, cospi_11_64); const __m128i stg1_12 = pair_set_epi16(cospi_19_64, -cospi_13_64); const __m128i stg1_13 = pair_set_epi16(cospi_13_64, cospi_19_64); const __m128i stg1_14 = pair_set_epi16(cospi_3_64, -cospi_29_64); const __m128i stg1_15 = pair_set_epi16(cospi_29_64, cospi_3_64); const __m128i stg2_0 = pair_set_epi16(cospi_30_64, -cospi_2_64); const __m128i stg2_1 = pair_set_epi16(cospi_2_64, cospi_30_64); const __m128i stg2_2 = pair_set_epi16(cospi_14_64, -cospi_18_64); const __m128i stg2_3 = pair_set_epi16(cospi_18_64, cospi_14_64); const __m128i stg2_4 = pair_set_epi16(cospi_22_64, -cospi_10_64); const __m128i stg2_5 = pair_set_epi16(cospi_10_64, cospi_22_64); const __m128i stg2_6 = pair_set_epi16(cospi_6_64, -cospi_26_64); const __m128i stg2_7 = pair_set_epi16(cospi_26_64, cospi_6_64); const __m128i stg3_0 = pair_set_epi16(cospi_28_64, -cospi_4_64); const __m128i stg3_1 = pair_set_epi16(cospi_4_64, cospi_28_64); const __m128i stg3_2 = pair_set_epi16(cospi_12_64, -cospi_20_64); const __m128i stg3_3 = pair_set_epi16(cospi_20_64, cospi_12_64); const __m128i stg3_4 = pair_set_epi16(-cospi_4_64, cospi_28_64); const __m128i stg3_5 = pair_set_epi16(cospi_28_64, cospi_4_64); const __m128i stg3_6 = pair_set_epi16(-cospi_28_64, -cospi_4_64); const __m128i stg3_8 = pair_set_epi16(-cospi_20_64, cospi_12_64); const __m128i stg3_9 = pair_set_epi16(cospi_12_64, cospi_20_64); const __m128i stg3_10 = pair_set_epi16(-cospi_12_64, -cospi_20_64); const __m128i stg4_0 = pair_set_epi16(cospi_16_64, cospi_16_64); const __m128i stg4_1 = pair_set_epi16(cospi_16_64, -cospi_16_64); const __m128i stg4_2 = pair_set_epi16(cospi_24_64, -cospi_8_64); const __m128i stg4_3 = pair_set_epi16(cospi_8_64, cospi_24_64); const __m128i stg4_4 = pair_set_epi16(-cospi_8_64, cospi_24_64); const __m128i stg4_5 = pair_set_epi16(cospi_24_64, cospi_8_64); const __m128i stg4_6 = pair_set_epi16(-cospi_24_64, -cospi_8_64); const __m128i stg6_0 = pair_set_epi16(-cospi_16_64, cospi_16_64); __m128i stp1_0, stp1_1, stp1_2, stp1_3, stp1_4, stp1_5, stp1_6, stp1_7, stp1_8, stp1_9, stp1_10, stp1_11, stp1_12, stp1_13, stp1_14, stp1_15, stp1_16, stp1_17, stp1_18, stp1_19, stp1_20, stp1_21, stp1_22, stp1_23, stp1_24, stp1_25, stp1_26, stp1_27, stp1_28, stp1_29, stp1_30, stp1_31; __m128i stp2_0, stp2_1, stp2_2, stp2_3, stp2_4, stp2_5, stp2_6, stp2_7, stp2_8, stp2_9, stp2_10, stp2_11, stp2_12, stp2_13, stp2_14, stp2_15, stp2_16, stp2_17, stp2_18, stp2_19, stp2_20, stp2_21, stp2_22, stp2_23, stp2_24, stp2_25, stp2_26, stp2_27, stp2_28, stp2_29, stp2_30, stp2_31; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; /* Stage1 */ { const __m128i lo_1_31 = _mm_unpacklo_epi16(in[1], in[31]); const __m128i hi_1_31 = _mm_unpackhi_epi16(in[1], in[31]); const __m128i lo_17_15 = _mm_unpacklo_epi16(in[17], in[15]); const __m128i hi_17_15 = _mm_unpackhi_epi16(in[17], in[15]); const __m128i lo_9_23 = _mm_unpacklo_epi16(in[9], in[23]); const __m128i hi_9_23 = _mm_unpackhi_epi16(in[9], in[23]); const __m128i lo_25_7 = _mm_unpacklo_epi16(in[25], in[7]); const __m128i hi_25_7 = _mm_unpackhi_epi16(in[25], in[7]); const __m128i lo_5_27 = _mm_unpacklo_epi16(in[5], in[27]); const __m128i hi_5_27 = _mm_unpackhi_epi16(in[5], in[27]); const __m128i lo_21_11 = _mm_unpacklo_epi16(in[21], in[11]); const __m128i hi_21_11 = _mm_unpackhi_epi16(in[21], in[11]); const __m128i lo_13_19 = _mm_unpacklo_epi16(in[13], in[19]); const __m128i hi_13_19 = _mm_unpackhi_epi16(in[13], in[19]); const __m128i lo_29_3 = _mm_unpacklo_epi16(in[29], in[3]); const __m128i hi_29_3 = _mm_unpackhi_epi16(in[29], in[3]); MULTIPLICATION_AND_ADD(lo_1_31, hi_1_31, lo_17_15, hi_17_15, stg1_0, stg1_1, stg1_2, stg1_3, stp1_16, stp1_31, stp1_17, stp1_30) MULTIPLICATION_AND_ADD(lo_9_23, hi_9_23, lo_25_7, hi_25_7, stg1_4, stg1_5, stg1_6, stg1_7, stp1_18, stp1_29, stp1_19, stp1_28) MULTIPLICATION_AND_ADD(lo_5_27, hi_5_27, lo_21_11, hi_21_11, stg1_8, stg1_9, stg1_10, stg1_11, stp1_20, stp1_27, stp1_21, stp1_26) MULTIPLICATION_AND_ADD(lo_13_19, hi_13_19, lo_29_3, hi_29_3, stg1_12, stg1_13, stg1_14, stg1_15, stp1_22, stp1_25, stp1_23, stp1_24) } /* Stage2 */ { const __m128i lo_2_30 = _mm_unpacklo_epi16(in[2], in[30]); const __m128i hi_2_30 = _mm_unpackhi_epi16(in[2], in[30]); const __m128i lo_18_14 = _mm_unpacklo_epi16(in[18], in[14]); const __m128i hi_18_14 = _mm_unpackhi_epi16(in[18], in[14]); const __m128i lo_10_22 = _mm_unpacklo_epi16(in[10], in[22]); const __m128i hi_10_22 = _mm_unpackhi_epi16(in[10], in[22]); const __m128i lo_26_6 = _mm_unpacklo_epi16(in[26], in[6]); const __m128i hi_26_6 = _mm_unpackhi_epi16(in[26], in[6]); MULTIPLICATION_AND_ADD(lo_2_30, hi_2_30, lo_18_14, hi_18_14, stg2_0, stg2_1, stg2_2, stg2_3, stp2_8, stp2_15, stp2_9, stp2_14) MULTIPLICATION_AND_ADD(lo_10_22, hi_10_22, lo_26_6, hi_26_6, stg2_4, stg2_5, stg2_6, stg2_7, stp2_10, stp2_13, stp2_11, stp2_12) stp2_16 = _mm_add_epi16(stp1_16, stp1_17); stp2_17 = _mm_sub_epi16(stp1_16, stp1_17); stp2_18 = _mm_sub_epi16(stp1_19, stp1_18); stp2_19 = _mm_add_epi16(stp1_19, stp1_18); stp2_20 = _mm_add_epi16(stp1_20, stp1_21); stp2_21 = _mm_sub_epi16(stp1_20, stp1_21); stp2_22 = _mm_sub_epi16(stp1_23, stp1_22); stp2_23 = _mm_add_epi16(stp1_23, stp1_22); stp2_24 = _mm_add_epi16(stp1_24, stp1_25); stp2_25 = _mm_sub_epi16(stp1_24, stp1_25); stp2_26 = _mm_sub_epi16(stp1_27, stp1_26); stp2_27 = _mm_add_epi16(stp1_27, stp1_26); stp2_28 = _mm_add_epi16(stp1_28, stp1_29); stp2_29 = _mm_sub_epi16(stp1_28, stp1_29); stp2_30 = _mm_sub_epi16(stp1_31, stp1_30); stp2_31 = _mm_add_epi16(stp1_31, stp1_30); } /* Stage3 */ { const __m128i lo_4_28 = _mm_unpacklo_epi16(in[4], in[28]); const __m128i hi_4_28 = _mm_unpackhi_epi16(in[4], in[28]); const __m128i lo_20_12 = _mm_unpacklo_epi16(in[20], in[12]); const __m128i hi_20_12 = _mm_unpackhi_epi16(in[20], in[12]); const __m128i lo_17_30 = _mm_unpacklo_epi16(stp2_17, stp2_30); const __m128i hi_17_30 = _mm_unpackhi_epi16(stp2_17, stp2_30); const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); MULTIPLICATION_AND_ADD(lo_4_28, hi_4_28, lo_20_12, hi_20_12, stg3_0, stg3_1, stg3_2, stg3_3, stp1_4, stp1_7, stp1_5, stp1_6) stp1_8 = _mm_add_epi16(stp2_8, stp2_9); stp1_9 = _mm_sub_epi16(stp2_8, stp2_9); stp1_10 = _mm_sub_epi16(stp2_11, stp2_10); stp1_11 = _mm_add_epi16(stp2_11, stp2_10); stp1_12 = _mm_add_epi16(stp2_12, stp2_13); stp1_13 = _mm_sub_epi16(stp2_12, stp2_13); stp1_14 = _mm_sub_epi16(stp2_15, stp2_14); stp1_15 = _mm_add_epi16(stp2_15, stp2_14); MULTIPLICATION_AND_ADD(lo_17_30, hi_17_30, lo_18_29, hi_18_29, stg3_4, stg3_5, stg3_6, stg3_4, stp1_17, stp1_30, stp1_18, stp1_29) MULTIPLICATION_AND_ADD(lo_21_26, hi_21_26, lo_22_25, hi_22_25, stg3_8, stg3_9, stg3_10, stg3_8, stp1_21, stp1_26, stp1_22, stp1_25) stp1_16 = stp2_16; stp1_31 = stp2_31; stp1_19 = stp2_19; stp1_20 = stp2_20; stp1_23 = stp2_23; stp1_24 = stp2_24; stp1_27 = stp2_27; stp1_28 = stp2_28; } /* Stage4 */ { const __m128i lo_0_16 = _mm_unpacklo_epi16(in[0], in[16]); const __m128i hi_0_16 = _mm_unpackhi_epi16(in[0], in[16]); const __m128i lo_8_24 = _mm_unpacklo_epi16(in[8], in[24]); const __m128i hi_8_24 = _mm_unpackhi_epi16(in[8], in[24]); const __m128i lo_9_14 = _mm_unpacklo_epi16(stp1_9, stp1_14); const __m128i hi_9_14 = _mm_unpackhi_epi16(stp1_9, stp1_14); const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); MULTIPLICATION_AND_ADD(lo_0_16, hi_0_16, lo_8_24, hi_8_24, stg4_0, stg4_1, stg4_2, stg4_3, stp2_0, stp2_1, stp2_2, stp2_3) stp2_4 = _mm_add_epi16(stp1_4, stp1_5); stp2_5 = _mm_sub_epi16(stp1_4, stp1_5); stp2_6 = _mm_sub_epi16(stp1_7, stp1_6); stp2_7 = _mm_add_epi16(stp1_7, stp1_6); MULTIPLICATION_AND_ADD(lo_9_14, hi_9_14, lo_10_13, hi_10_13, stg4_4, stg4_5, stg4_6, stg4_4, stp2_9, stp2_14, stp2_10, stp2_13) stp2_8 = stp1_8; stp2_15 = stp1_15; stp2_11 = stp1_11; stp2_12 = stp1_12; stp2_16 = _mm_add_epi16(stp1_16, stp1_19); stp2_17 = _mm_add_epi16(stp1_17, stp1_18); stp2_18 = _mm_sub_epi16(stp1_17, stp1_18); stp2_19 = _mm_sub_epi16(stp1_16, stp1_19); stp2_20 = _mm_sub_epi16(stp1_23, stp1_20); stp2_21 = _mm_sub_epi16(stp1_22, stp1_21); stp2_22 = _mm_add_epi16(stp1_22, stp1_21); stp2_23 = _mm_add_epi16(stp1_23, stp1_20); stp2_24 = _mm_add_epi16(stp1_24, stp1_27); stp2_25 = _mm_add_epi16(stp1_25, stp1_26); stp2_26 = _mm_sub_epi16(stp1_25, stp1_26); stp2_27 = _mm_sub_epi16(stp1_24, stp1_27); stp2_28 = _mm_sub_epi16(stp1_31, stp1_28); stp2_29 = _mm_sub_epi16(stp1_30, stp1_29); stp2_30 = _mm_add_epi16(stp1_29, stp1_30); stp2_31 = _mm_add_epi16(stp1_28, stp1_31); } /* Stage5 */ { const __m128i lo_6_5 = _mm_unpacklo_epi16(stp2_6, stp2_5); const __m128i hi_6_5 = _mm_unpackhi_epi16(stp2_6, stp2_5); const __m128i lo_18_29 = _mm_unpacklo_epi16(stp2_18, stp2_29); const __m128i hi_18_29 = _mm_unpackhi_epi16(stp2_18, stp2_29); const __m128i lo_19_28 = _mm_unpacklo_epi16(stp2_19, stp2_28); const __m128i hi_19_28 = _mm_unpackhi_epi16(stp2_19, stp2_28); const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); stp1_0 = _mm_add_epi16(stp2_0, stp2_3); stp1_1 = _mm_add_epi16(stp2_1, stp2_2); stp1_2 = _mm_sub_epi16(stp2_1, stp2_2); stp1_3 = _mm_sub_epi16(stp2_0, stp2_3); tmp0 = _mm_madd_epi16(lo_6_5, stg4_1); tmp1 = _mm_madd_epi16(hi_6_5, stg4_1); tmp2 = _mm_madd_epi16(lo_6_5, stg4_0); tmp3 = _mm_madd_epi16(hi_6_5, stg4_0); tmp0 = _mm_add_epi32(tmp0, rounding); tmp1 = _mm_add_epi32(tmp1, rounding); tmp2 = _mm_add_epi32(tmp2, rounding); tmp3 = _mm_add_epi32(tmp3, rounding); tmp0 = _mm_srai_epi32(tmp0, DCT_CONST_BITS); tmp1 = _mm_srai_epi32(tmp1, DCT_CONST_BITS); tmp2 = _mm_srai_epi32(tmp2, DCT_CONST_BITS); tmp3 = _mm_srai_epi32(tmp3, DCT_CONST_BITS); stp1_5 = _mm_packs_epi32(tmp0, tmp1); stp1_6 = _mm_packs_epi32(tmp2, tmp3); stp1_4 = stp2_4; stp1_7 = stp2_7; stp1_8 = _mm_add_epi16(stp2_8, stp2_11); stp1_9 = _mm_add_epi16(stp2_9, stp2_10); stp1_10 = _mm_sub_epi16(stp2_9, stp2_10); stp1_11 = _mm_sub_epi16(stp2_8, stp2_11); stp1_12 = _mm_sub_epi16(stp2_15, stp2_12); stp1_13 = _mm_sub_epi16(stp2_14, stp2_13); stp1_14 = _mm_add_epi16(stp2_14, stp2_13); stp1_15 = _mm_add_epi16(stp2_15, stp2_12); stp1_16 = stp2_16; stp1_17 = stp2_17; MULTIPLICATION_AND_ADD(lo_18_29, hi_18_29, lo_19_28, hi_19_28, stg4_4, stg4_5, stg4_4, stg4_5, stp1_18, stp1_29, stp1_19, stp1_28) MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg4_6, stg4_4, stg4_6, stg4_4, stp1_20, stp1_27, stp1_21, stp1_26) stp1_22 = stp2_22; stp1_23 = stp2_23; stp1_24 = stp2_24; stp1_25 = stp2_25; stp1_30 = stp2_30; stp1_31 = stp2_31; } /* Stage6 */ { const __m128i lo_10_13 = _mm_unpacklo_epi16(stp1_10, stp1_13); const __m128i hi_10_13 = _mm_unpackhi_epi16(stp1_10, stp1_13); const __m128i lo_11_12 = _mm_unpacklo_epi16(stp1_11, stp1_12); const __m128i hi_11_12 = _mm_unpackhi_epi16(stp1_11, stp1_12); stp2_0 = _mm_add_epi16(stp1_0, stp1_7); stp2_1 = _mm_add_epi16(stp1_1, stp1_6); stp2_2 = _mm_add_epi16(stp1_2, stp1_5); stp2_3 = _mm_add_epi16(stp1_3, stp1_4); stp2_4 = _mm_sub_epi16(stp1_3, stp1_4); stp2_5 = _mm_sub_epi16(stp1_2, stp1_5); stp2_6 = _mm_sub_epi16(stp1_1, stp1_6); stp2_7 = _mm_sub_epi16(stp1_0, stp1_7); stp2_8 = stp1_8; stp2_9 = stp1_9; stp2_14 = stp1_14; stp2_15 = stp1_15; MULTIPLICATION_AND_ADD(lo_10_13, hi_10_13, lo_11_12, hi_11_12, stg6_0, stg4_0, stg6_0, stg4_0, stp2_10, stp2_13, stp2_11, stp2_12) stp2_16 = _mm_add_epi16(stp1_16, stp1_23); stp2_17 = _mm_add_epi16(stp1_17, stp1_22); stp2_18 = _mm_add_epi16(stp1_18, stp1_21); stp2_19 = _mm_add_epi16(stp1_19, stp1_20); stp2_20 = _mm_sub_epi16(stp1_19, stp1_20); stp2_21 = _mm_sub_epi16(stp1_18, stp1_21); stp2_22 = _mm_sub_epi16(stp1_17, stp1_22); stp2_23 = _mm_sub_epi16(stp1_16, stp1_23); stp2_24 = _mm_sub_epi16(stp1_31, stp1_24); stp2_25 = _mm_sub_epi16(stp1_30, stp1_25); stp2_26 = _mm_sub_epi16(stp1_29, stp1_26); stp2_27 = _mm_sub_epi16(stp1_28, stp1_27); stp2_28 = _mm_add_epi16(stp1_27, stp1_28); stp2_29 = _mm_add_epi16(stp1_26, stp1_29); stp2_30 = _mm_add_epi16(stp1_25, stp1_30); stp2_31 = _mm_add_epi16(stp1_24, stp1_31); } /* Stage7 */ { const __m128i lo_20_27 = _mm_unpacklo_epi16(stp2_20, stp2_27); const __m128i hi_20_27 = _mm_unpackhi_epi16(stp2_20, stp2_27); const __m128i lo_21_26 = _mm_unpacklo_epi16(stp2_21, stp2_26); const __m128i hi_21_26 = _mm_unpackhi_epi16(stp2_21, stp2_26); const __m128i lo_22_25 = _mm_unpacklo_epi16(stp2_22, stp2_25); const __m128i hi_22_25 = _mm_unpackhi_epi16(stp2_22, stp2_25); const __m128i lo_23_24 = _mm_unpacklo_epi16(stp2_23, stp2_24); const __m128i hi_23_24 = _mm_unpackhi_epi16(stp2_23, stp2_24); stp1_0 = _mm_add_epi16(stp2_0, stp2_15); stp1_1 = _mm_add_epi16(stp2_1, stp2_14); stp1_2 = _mm_add_epi16(stp2_2, stp2_13); stp1_3 = _mm_add_epi16(stp2_3, stp2_12); stp1_4 = _mm_add_epi16(stp2_4, stp2_11); stp1_5 = _mm_add_epi16(stp2_5, stp2_10); stp1_6 = _mm_add_epi16(stp2_6, stp2_9); stp1_7 = _mm_add_epi16(stp2_7, stp2_8); stp1_8 = _mm_sub_epi16(stp2_7, stp2_8); stp1_9 = _mm_sub_epi16(stp2_6, stp2_9); stp1_10 = _mm_sub_epi16(stp2_5, stp2_10); stp1_11 = _mm_sub_epi16(stp2_4, stp2_11); stp1_12 = _mm_sub_epi16(stp2_3, stp2_12); stp1_13 = _mm_sub_epi16(stp2_2, stp2_13); stp1_14 = _mm_sub_epi16(stp2_1, stp2_14); stp1_15 = _mm_sub_epi16(stp2_0, stp2_15); stp1_16 = stp2_16; stp1_17 = stp2_17; stp1_18 = stp2_18; stp1_19 = stp2_19; MULTIPLICATION_AND_ADD(lo_20_27, hi_20_27, lo_21_26, hi_21_26, stg6_0, stg4_0, stg6_0, stg4_0, stp1_20, stp1_27, stp1_21, stp1_26) MULTIPLICATION_AND_ADD(lo_22_25, hi_22_25, lo_23_24, hi_23_24, stg6_0, stg4_0, stg6_0, stg4_0, stp1_22, stp1_25, stp1_23, stp1_24) stp1_28 = stp2_28; stp1_29 = stp2_29; stp1_30 = stp2_30; stp1_31 = stp2_31; } out[0] = _mm_add_epi16(stp1_0, stp1_31); out[1] = _mm_add_epi16(stp1_1, stp1_30); out[2] = _mm_add_epi16(stp1_2, stp1_29); out[3] = _mm_add_epi16(stp1_3, stp1_28); out[4] = _mm_add_epi16(stp1_4, stp1_27); out[5] = _mm_add_epi16(stp1_5, stp1_26); out[6] = _mm_add_epi16(stp1_6, stp1_25); out[7] = _mm_add_epi16(stp1_7, stp1_24); out[8] = _mm_add_epi16(stp1_8, stp1_23); out[9] = _mm_add_epi16(stp1_9, stp1_22); out[10] = _mm_add_epi16(stp1_10, stp1_21); out[11] = _mm_add_epi16(stp1_11, stp1_20); out[12] = _mm_add_epi16(stp1_12, stp1_19); out[13] = _mm_add_epi16(stp1_13, stp1_18); out[14] = _mm_add_epi16(stp1_14, stp1_17); out[15] = _mm_add_epi16(stp1_15, stp1_16); out[16] = _mm_sub_epi16(stp1_15, stp1_16); out[17] = _mm_sub_epi16(stp1_14, stp1_17); out[18] = _mm_sub_epi16(stp1_13, stp1_18); out[19] = _mm_sub_epi16(stp1_12, stp1_19); out[20] = _mm_sub_epi16(stp1_11, stp1_20); out[21] = _mm_sub_epi16(stp1_10, stp1_21); out[22] = _mm_sub_epi16(stp1_9, stp1_22); out[23] = _mm_sub_epi16(stp1_8, stp1_23); out[24] = _mm_sub_epi16(stp1_7, stp1_24); out[25] = _mm_sub_epi16(stp1_6, stp1_25); out[26] = _mm_sub_epi16(stp1_5, stp1_26); out[27] = _mm_sub_epi16(stp1_4, stp1_27); out[28] = _mm_sub_epi16(stp1_3, stp1_28); out[29] = _mm_sub_epi16(stp1_2, stp1_29); out[30] = _mm_sub_epi16(stp1_1, stp1_30); out[31] = _mm_sub_epi16(stp1_0, stp1_31); } static void load_buffer_8x32(const tran_low_t *input, __m128i *in) { int i; for (i = 0; i < 8; ++i) { in[i] = load_input_data(input); in[i + 8] = load_input_data(input + 8); in[i + 16] = load_input_data(input + 16); in[i + 24] = load_input_data(input + 24); input += 32; } } void vpx_idct32x32_1024_add_ssse3(const tran_low_t *input, uint8_t *dest, int stride) { __m128i col[128], in[32]; int i, j; // rows for (i = 0; i < 4; ++i) { load_buffer_8x32(input, in); input += 32 << 3; // Transpose 32x8 block to 8x32 block array_transpose_8x8(in, in); array_transpose_8x8(in + 8, in + 8); array_transpose_8x8(in + 16, in + 16); array_transpose_8x8(in + 24, in + 24); idct32_8x32(in, col + (i << 5)); } // columns for (i = 0; i < 4; ++i) { j = i << 3; // Transpose 32x8 block to 8x32 block array_transpose_8x8(col + j, in); array_transpose_8x8(col + j + 32, in + 8); array_transpose_8x8(col + j + 64, in + 16); array_transpose_8x8(col + j + 96, in + 24); idct32_8x32(in, in); store_buffer_8x32(in, dest, stride); dest += 8; } }