ref: 82d504b50f5dbc81ba1e1e1c1b07bb76dddde43f
parent: a32a086d23c2061344af7653892456bde3fffd0d
author: Jingning Han <[email protected]>
date: Mon Jun 24 15:52:55 EDT 2013
Use aligned buffer operations in 8x8/16x16 2D-DCT This reduces 16x16 2D-DCT runtime from 865 cycles to 837 cycles. Change-Id: I137758b81cd127b936175284310e81378db64552
--- a/vp9/encoder/x86/vp9_dct_sse2.c
+++ b/vp9/encoder/x86/vp9_dct_sse2.c
@@ -134,14 +134,14 @@
const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
// Load input
- __m128i in0 = _mm_loadu_si128((const __m128i *)(input + 0 * stride));
- __m128i in1 = _mm_loadu_si128((const __m128i *)(input + 1 * stride));
- __m128i in2 = _mm_loadu_si128((const __m128i *)(input + 2 * stride));
- __m128i in3 = _mm_loadu_si128((const __m128i *)(input + 3 * stride));
- __m128i in4 = _mm_loadu_si128((const __m128i *)(input + 4 * stride));
- __m128i in5 = _mm_loadu_si128((const __m128i *)(input + 5 * stride));
- __m128i in6 = _mm_loadu_si128((const __m128i *)(input + 6 * stride));
- __m128i in7 = _mm_loadu_si128((const __m128i *)(input + 7 * stride));
+ __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
+ __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
+ __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
+ __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
+ __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
+ __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
+ __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
+ __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
// Pre-condition input (shift by two)
in0 = _mm_slli_epi16(in0, 2);
in1 = _mm_slli_epi16(in1, 2);
@@ -363,14 +363,14 @@
in6 = _mm_srai_epi16(in6, 1);
in7 = _mm_srai_epi16(in7, 1);
// store results
- _mm_storeu_si128((__m128i *)(output + 0 * 8), in0);
- _mm_storeu_si128((__m128i *)(output + 1 * 8), in1);
- _mm_storeu_si128((__m128i *)(output + 2 * 8), in2);
- _mm_storeu_si128((__m128i *)(output + 3 * 8), in3);
- _mm_storeu_si128((__m128i *)(output + 4 * 8), in4);
- _mm_storeu_si128((__m128i *)(output + 5 * 8), in5);
- _mm_storeu_si128((__m128i *)(output + 6 * 8), in6);
- _mm_storeu_si128((__m128i *)(output + 7 * 8), in7);
+ _mm_store_si128((__m128i *)(output + 0 * 8), in0);
+ _mm_store_si128((__m128i *)(output + 1 * 8), in1);
+ _mm_store_si128((__m128i *)(output + 2 * 8), in2);
+ _mm_store_si128((__m128i *)(output + 3 * 8), in3);
+ _mm_store_si128((__m128i *)(output + 4 * 8), in4);
+ _mm_store_si128((__m128i *)(output + 5 * 8), in5);
+ _mm_store_si128((__m128i *)(output + 6 * 8), in6);
+ _mm_store_si128((__m128i *)(output + 7 * 8), in7);
}
}
@@ -876,7 +876,7 @@
const int stride = pitch >> 1;
int pass;
// We need an intermediate buffer between passes.
- int16_t intermediate[256];
+ DECLARE_ALIGNED_ARRAY(16, int16_t, intermediate, 256);
int16_t *in = input;
int16_t *out = intermediate;
// Constants
@@ -919,22 +919,22 @@
__m128i res08, res09, res10, res11, res12, res13, res14, res15;
// Load and pre-condition input.
if (0 == pass) {- in00 = _mm_loadu_si128((const __m128i *)(in + 0 * stride));
- in01 = _mm_loadu_si128((const __m128i *)(in + 1 * stride));
- in02 = _mm_loadu_si128((const __m128i *)(in + 2 * stride));
- in03 = _mm_loadu_si128((const __m128i *)(in + 3 * stride));
- in04 = _mm_loadu_si128((const __m128i *)(in + 4 * stride));
- in05 = _mm_loadu_si128((const __m128i *)(in + 5 * stride));
- in06 = _mm_loadu_si128((const __m128i *)(in + 6 * stride));
- in07 = _mm_loadu_si128((const __m128i *)(in + 7 * stride));
- in08 = _mm_loadu_si128((const __m128i *)(in + 8 * stride));
- in09 = _mm_loadu_si128((const __m128i *)(in + 9 * stride));
- in10 = _mm_loadu_si128((const __m128i *)(in + 10 * stride));
- in11 = _mm_loadu_si128((const __m128i *)(in + 11 * stride));
- in12 = _mm_loadu_si128((const __m128i *)(in + 12 * stride));
- in13 = _mm_loadu_si128((const __m128i *)(in + 13 * stride));
- in14 = _mm_loadu_si128((const __m128i *)(in + 14 * stride));
- in15 = _mm_loadu_si128((const __m128i *)(in + 15 * stride));
+ in00 = _mm_load_si128((const __m128i *)(in + 0 * stride));
+ in01 = _mm_load_si128((const __m128i *)(in + 1 * stride));
+ in02 = _mm_load_si128((const __m128i *)(in + 2 * stride));
+ in03 = _mm_load_si128((const __m128i *)(in + 3 * stride));
+ in04 = _mm_load_si128((const __m128i *)(in + 4 * stride));
+ in05 = _mm_load_si128((const __m128i *)(in + 5 * stride));
+ in06 = _mm_load_si128((const __m128i *)(in + 6 * stride));
+ in07 = _mm_load_si128((const __m128i *)(in + 7 * stride));
+ in08 = _mm_load_si128((const __m128i *)(in + 8 * stride));
+ in09 = _mm_load_si128((const __m128i *)(in + 9 * stride));
+ in10 = _mm_load_si128((const __m128i *)(in + 10 * stride));
+ in11 = _mm_load_si128((const __m128i *)(in + 11 * stride));
+ in12 = _mm_load_si128((const __m128i *)(in + 12 * stride));
+ in13 = _mm_load_si128((const __m128i *)(in + 13 * stride));
+ in14 = _mm_load_si128((const __m128i *)(in + 14 * stride));
+ in15 = _mm_load_si128((const __m128i *)(in + 15 * stride));
// x = x << 2
in00 = _mm_slli_epi16(in00, 2);
in01 = _mm_slli_epi16(in01, 2);
@@ -953,22 +953,22 @@
in14 = _mm_slli_epi16(in14, 2);
in15 = _mm_slli_epi16(in15, 2);
} else {- in00 = _mm_loadu_si128((const __m128i *)(in + 0 * 16));
- in01 = _mm_loadu_si128((const __m128i *)(in + 1 * 16));
- in02 = _mm_loadu_si128((const __m128i *)(in + 2 * 16));
- in03 = _mm_loadu_si128((const __m128i *)(in + 3 * 16));
- in04 = _mm_loadu_si128((const __m128i *)(in + 4 * 16));
- in05 = _mm_loadu_si128((const __m128i *)(in + 5 * 16));
- in06 = _mm_loadu_si128((const __m128i *)(in + 6 * 16));
- in07 = _mm_loadu_si128((const __m128i *)(in + 7 * 16));
- in08 = _mm_loadu_si128((const __m128i *)(in + 8 * 16));
- in09 = _mm_loadu_si128((const __m128i *)(in + 9 * 16));
- in10 = _mm_loadu_si128((const __m128i *)(in + 10 * 16));
- in11 = _mm_loadu_si128((const __m128i *)(in + 11 * 16));
- in12 = _mm_loadu_si128((const __m128i *)(in + 12 * 16));
- in13 = _mm_loadu_si128((const __m128i *)(in + 13 * 16));
- in14 = _mm_loadu_si128((const __m128i *)(in + 14 * 16));
- in15 = _mm_loadu_si128((const __m128i *)(in + 15 * 16));
+ in00 = _mm_load_si128((const __m128i *)(in + 0 * 16));
+ in01 = _mm_load_si128((const __m128i *)(in + 1 * 16));
+ in02 = _mm_load_si128((const __m128i *)(in + 2 * 16));
+ in03 = _mm_load_si128((const __m128i *)(in + 3 * 16));
+ in04 = _mm_load_si128((const __m128i *)(in + 4 * 16));
+ in05 = _mm_load_si128((const __m128i *)(in + 5 * 16));
+ in06 = _mm_load_si128((const __m128i *)(in + 6 * 16));
+ in07 = _mm_load_si128((const __m128i *)(in + 7 * 16));
+ in08 = _mm_load_si128((const __m128i *)(in + 8 * 16));
+ in09 = _mm_load_si128((const __m128i *)(in + 9 * 16));
+ in10 = _mm_load_si128((const __m128i *)(in + 10 * 16));
+ in11 = _mm_load_si128((const __m128i *)(in + 11 * 16));
+ in12 = _mm_load_si128((const __m128i *)(in + 12 * 16));
+ in13 = _mm_load_si128((const __m128i *)(in + 13 * 16));
+ in14 = _mm_load_si128((const __m128i *)(in + 14 * 16));
+ in15 = _mm_load_si128((const __m128i *)(in + 15 * 16));
// x = (x + 1) >> 2
in00 = _mm_add_epi16(in00, kOne);
in01 = _mm_add_epi16(in01, kOne);
@@ -1475,14 +1475,14 @@
// 06 16 26 36 46 56 66 76
// 07 17 27 37 47 57 67 77
// Store results
- _mm_storeu_si128((__m128i *)(out + 8 + 0 * 16), tr2_0);
- _mm_storeu_si128((__m128i *)(out + 8 + 1 * 16), tr2_1);
- _mm_storeu_si128((__m128i *)(out + 8 + 2 * 16), tr2_2);
- _mm_storeu_si128((__m128i *)(out + 8 + 3 * 16), tr2_3);
- _mm_storeu_si128((__m128i *)(out + 8 + 4 * 16), tr2_4);
- _mm_storeu_si128((__m128i *)(out + 8 + 5 * 16), tr2_5);
- _mm_storeu_si128((__m128i *)(out + 8 + 6 * 16), tr2_6);
- _mm_storeu_si128((__m128i *)(out + 8 + 7 * 16), tr2_7);
+ _mm_store_si128((__m128i *)(out + 8 + 0 * 16), tr2_0);
+ _mm_store_si128((__m128i *)(out + 8 + 1 * 16), tr2_1);
+ _mm_store_si128((__m128i *)(out + 8 + 2 * 16), tr2_2);
+ _mm_store_si128((__m128i *)(out + 8 + 3 * 16), tr2_3);
+ _mm_store_si128((__m128i *)(out + 8 + 4 * 16), tr2_4);
+ _mm_store_si128((__m128i *)(out + 8 + 5 * 16), tr2_5);
+ _mm_store_si128((__m128i *)(out + 8 + 6 * 16), tr2_6);
+ _mm_store_si128((__m128i *)(out + 8 + 7 * 16), tr2_7);
}
out += 8*16;
}