ref: 3a6ed92a351a5117d3e415bc39e7bd08f6ed6f79
dir: /codec/encoder/core/arm64/reconstruct_aarch64_neon.S/
/*! * \copy * Copyright (c) 2013, Cisco Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * */ #ifdef HAVE_NEON_AARCH64 #include "arm_arch64_common_macro.S" .macro ZERO_COUNT_IN_2_QUARWORD arg0, arg1, arg2 // { // input: coef_0 (identy to \arg3\() \arg4\()), coef_1(identy to \arg5\() \arg6\()), mask_q cmeq \arg0\().8h, \arg0\().8h, #0 cmeq \arg1\().8h, \arg1\().8h, #0 uzp1 \arg0\().16b, \arg0\().16b, \arg1\().16b ushr \arg0\().16b, \arg0\().16b, 7 addv \arg2\(), \arg0\().16b // } .endm .macro NEWQUANT_COEF_EACH_16BITS arg0, arg1, arg2, arg3, arg4, arg5 // if coef <= 0, - coef; else , coef; // { // input: coef, ff (dst), mf eor \arg3\().16b, \arg3\().16b, \arg3\().16b // init 0 , and keep 0; saba \arg1\().8h, \arg0\().8h, \arg3\().8h // f + abs(coef - 0) smull \arg4\().4s, \arg1\().4h, \arg2\().4h smull2 \arg5\().4s, \arg1\().8h, \arg2\().8h shrn \arg1\().4h, \arg4\().4s, #16 shrn2 \arg1\().8h, \arg5\().4s, #16 cmgt \arg4\().8h, \arg0\().8h, #0 // if true, location of coef == 11111111 bif \arg3\().16b, \arg1\().16b, \arg4\().16b // if (x<0) reserved part; else keep 0 untouched shl \arg3\().8h, \arg3\().8h, #1 sub \arg1\().8h, \arg1\().8h, \arg3\().8h // if x > 0, -= 0; else x-= 2x // } .endm .macro NEWQUANT_COEF_EACH_16BITS_MAX arg0, arg1, arg2, arg3, arg4, arg5, arg6 // if coef <= 0, - coef; else , coef; // { // input: coef, ff (dst), mf eor \arg3\().16b, \arg3\().16b, \arg3\().16b // init 0 , and keep 0; saba \arg1\().8h, \arg0\().8h, \arg3\().8h // f + abs(coef - 0) smull \arg4\().4s, \arg1\().4h, \arg2\().4h smull2 \arg5\().4s, \arg1\().8h, \arg2\().8h shrn \arg1\().4h, \arg4\().4s, #16 shrn2 \arg1\().8h, \arg5\().4s, #16 cmgt \arg4\().8h, \arg0\().8h, #0 // if true, location of coef == 11111111 bif \arg3\().16b, \arg1\().16b, \arg4\().16b // if (x<0) reserved part; else keep 0 untouched shl \arg3\().8h, \arg3\().8h, #1 mov \arg6\().16b, \arg1\().16b sub \arg1\().8h, \arg1\().8h, \arg3\().8h // if x > 0, -= 0; else x-= 2x // } .endm .macro QUANT_DUALWORD_COEF_EACH_16BITS arg0, arg1, arg2, arg3, arg4 // if coef <= 0, - coef; else , coef; // { // input: coef, ff (dst), mf saba \arg1\().8h, \arg0\().8h, \arg3\().8h // f + abs(coef - 0) smull \arg4\().4s, \arg1\().4h, \arg2\().4h shrn \arg1\().4h, \arg4\().4s, #16 cmgt \arg4\().8h, \arg0\().8h, #0 // if true, location of coef == 11111111 bif \arg3\().16b, \arg1\().16b, \arg4\().16b // if (x<0) reserved part; else keep 0 untouched shl \arg3\().8h, \arg3\().8h, #1 sub \arg1\().8h, \arg1\().8h, \arg3\().8h // if x > 0, -= 0; else x-= 2x // } .endm .macro SELECT_MAX_IN_ABS_COEF arg0, arg1, arg2, arg3, arg4, arg5 // { // input: coef_0, coef_1, coef_2, coef_3, max_q (identy to follow two) umax \arg0\().8h, \arg0\().8h, \arg1\().8h umaxv \arg4\(), \arg0\().8h umax \arg2\().8h, \arg2\().8h, \arg3\().8h umaxv \arg5\(), \arg2\().8h // } .endm .macro HDM_QUANT_2x2_TOTAL_16BITS arg0, arg1, arg2 // { // input: src_d[0][16][32][48], dst_d[0][16][32][48], working sshr \arg1\().2d, \arg0\().2d, #32 add \arg2\().4h, \arg0\().4h, \arg1\().4h // [0] = rs[0] + rs[32];[1] = rs[16] + rs[48]; sub \arg1\().4h, \arg0\().4h, \arg1\().4h // [0] = rs[0] - rs[32];[1] = rs[16] - rs[48]; zip1 \arg1\().4h, \arg2\().4h, \arg1\().4h // } .endm .macro DC_ZERO_COUNT_IN_DUALWORD arg0, arg1, arg2 // { // input: coef, dst_d, working_d (all 0x01) cmeq \arg0\().4h, \arg0\().4h, #0 and \arg0\().8b, \arg0\().8b, \arg2\().8b addv \arg1\(), \arg0\().4h // } .endm .macro IHDM_4x4_TOTAL_16BITS arg0, arg1, arg2 // { // input: each src_d[0]~[3](dst), working_q0, working_q1 uzp2 \arg1\().4s, \arg0\().4s, \arg0\().4s uzp1 \arg0\().4s, \arg0\().4s, \arg0\().4s add \arg2\().8h, \arg0\().8h, \arg1\().8h // [0] = rs[0] + rs[2];[1] = rs[1] + rs[3];[2] = rs[4] + rs[6];[3] = rs[5] + rs[7]; sub \arg1\().8h, \arg0\().8h, \arg1\().8h // [0] = rs[0] - rs[2];[1] = rs[1] - rs[3];[2] = rs[4] - rs[6];[3] = rs[5] - rs[7]; zip1 \arg2\().8h, \arg2\().8h, \arg1\().8h // [0] = rs[0] + rs[2]; [1] = rs[0] - rs[2]; ... [2]; [3] uzp2 \arg1\().4s, \arg2\().4s, \arg2\().4s uzp1 \arg0\().4s, \arg2\().4s, \arg2\().4s add \arg2\().8h, \arg0\().8h, \arg1\().8h // [0] = rs[0] + rs[2];[1] = rs[1] + rs[3];[2] = rs[4] + rs[6];[3] = rs[5] + rs[7]; sub \arg1\().8h, \arg0\().8h, \arg1\().8h // [0] = rs[0] - rs[2];[1] = rs[1] - rs[3];[2] = rs[4] - rs[6];[3] = rs[5] - rs[7]; rev32 \arg1\().4h, \arg1\().4h // [0] = rs[1] - rs[3];[1] = rs[0] - rs[2];[2] = rs[5] - rs[7];[3] = rs[4] - rs[6]; zip1 \arg0\().4s, \arg2\().4s, \arg1\().4s // } .endm .macro MATRIX_TRANSFORM_EACH_16BITS_2x8_OUT2 arg0, arg1, arg2, arg3 // { // input & output: src_d[0]~[3];[0 1 2 3]+[4 5 6 7]+[8 9 10 11]+[12 13 14 15] uzp1 \arg2\().4s, \arg0\().4s, \arg1\().4s //[0 1 4 5]+[8 9 12 13] uzp2 \arg3\().4s, \arg0\().4s, \arg1\().4s //[2 3 6 7]+[10 11 14 15] uzp1 \arg0\().8h, \arg2\().8h, \arg3\().8h //[0 4 8 12]+[2 6 10 14] uzp2 \arg2\().8h, \arg2\().8h, \arg3\().8h //[1 5 9 13]+[3 7 11 15] zip2 \arg1\().2d, \arg0\().2d, \arg2\().2d //[2 6 10 14]+[3 7 11 15] zip1 \arg0\().2d, \arg0\().2d, \arg2\().2d //[0 4 8 12]+[1 5 9 13] // } .endm .macro MATRIX_TRANSFORM_EACH_16BITS_OUT4 arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // { // input & output: src_d[0]~[3];[0 4 8 12],[1 5 9 13],[2 6 10 14],[3 7 11 15] trn1 \arg4\().8h, v0.8h, v1.8h trn2 \arg5\().8h, v0.8h, v1.8h trn1 \arg6\().8h, v2.8h, v3.8h trn2 \arg7\().8h, v2.8h, v3.8h trn1 \arg0\().4s, v4.4s, v6.4s trn2 \arg2\().4s, v4.4s, v6.4s trn1 \arg1\().4s, v5.4s, v7.4s trn2 \arg3\().4s, v5.4s, v7.4s // } .endm .macro MATRIX_TRANSFORM_EACH_16BITS_4x4_OUT2 arg0, arg1, arg2, arg3 // { // input & output: src_d[0]~[3];[0 1 2 3],[4 5 6 7],[8 9 10 11],[12 13 14 15] mov \arg0\().d[1], \arg1\().d[0] //[0 1 2 3]+[4 5 6 7] mov \arg2\().d[1], \arg3\().d[0] //[8 9 10 11]+[12 13 14 15] uzp1 \arg1\().4s, \arg0\().4s, \arg2\().4s //[0 1 4 5]+[8 9 12 13] uzp2 \arg3\().4s, \arg0\().4s, \arg2\().4s //[2 3 6 7]+[10 11 14 15] uzp1 \arg0\().8h, \arg1\().8h, \arg3\().8h //[0 4 8 12]+[2 6 10 14] uzp2 \arg2\().8h, \arg1\().8h, \arg3\().8h //[1 5 9 13]+[3 7 11 15] zip2 \arg1\().2d, \arg0\().2d, \arg2\().2d //[2 6 10 14]+[3 7 11 15] zip1 \arg0\().2d, \arg0\().2d, \arg2\().2d //[0 4 8 12]+[1 5 9 13] // } .endm .macro LOAD_4x4_DATA_FOR_DCT arg0, arg1, arg2, arg3, arg4, arg5 ld1 {\arg0\().s}[0], [\arg2\()], \arg3\() ld1 {\arg0\().s}[1], [\arg2\()], \arg3\() ld1 {\arg0\().s}[2], [\arg2\()], \arg3\() ld1 {\arg0\().s}[3], [\arg2\()] ld1 {\arg1\().s}[0], [\arg4\()], \arg5\() ld1 {\arg1\().s}[1], [\arg4\()], \arg5\() ld1 {\arg1\().s}[2], [\arg4\()], \arg5\() ld1 {\arg1\().s}[3], [\arg4\()] .endm .macro DCT_ROW_TRANSFORM_TOTAL_16BITS arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // { // input: src_d[0]~[3], working: [4]~[7] add \arg4\().8h, \arg0\().8h, \arg3\().8h //int16 s[0] = data[i] + data[i3]; sub \arg7\().8h, \arg0\().8h, \arg3\().8h //int16 s[3] = data[i] - data[i3]; add \arg5\().8h, \arg1\().8h, \arg2\().8h //int16 s[1] = data[i1] + data[i2]; sub \arg6\().8h, \arg1\().8h, \arg2\().8h //int16 s[2] = data[i1] - data[i2]; add \arg0\().8h, \arg4\().8h, \arg5\().8h //int16 dct[i ] = s[0] + s[1]; sub \arg2\().8h, \arg4\().8h, \arg5\().8h //int16 dct[i2] = s[0] - s[1]; shl \arg1\().8h, \arg7\().8h, #1 shl \arg3\().8h, \arg6\().8h, #1 add \arg1\().8h, \arg1\().8h, \arg6\().8h //int16 dct[i1] = (s[3] << 1) + s[2]; sub \arg3\().8h, \arg7\().8h, \arg3\().8h //int16 dct[i3] = s[3] - (s[2] << 1); // } .endm .macro LOAD_8x4_DATA_FOR_DCT arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9 // { // input: \arg0\()~\arg3\(), src1*, src2*; untouched r2:src1_stride &r4:src2_stride ld1 {\arg0\().d}[0], [\arg8\()], x2 ld1 {\arg1\().d}[0], [\arg8\()], x2 ld1 {\arg2\().d}[0], [\arg8\()], x2 ld1 {\arg3\().d}[0], [\arg8\()], x2 ld1 {\arg4\().d}[0], [\arg9\()], x4 ld1 {\arg5\().d}[0], [\arg9\()], x4 ld1 {\arg6\().d}[0], [\arg9\()], x4 ld1 {\arg7\().d}[0], [\arg9\()], x4 // } .endm .macro ROW_TRANSFORM_1_STEP_TOTAL_16BITS arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // { // input: src_d[0]~[3], output: e_d[0]~[3]; add \arg4\().8h, \arg0\().8h, \arg2\().8h //int16 e[i][0] = src[0] + src[2]; sub \arg5\().8h, \arg0\().8h, \arg2\().8h //int16 e[i][1] = src[0] - src[2]; sshr \arg6\().8h, \arg1\().8h, #1 sshr \arg7\().8h, \arg3\().8h, #1 sub \arg6\().8h, \arg6\().8h, \arg3\().8h //int16 e[i][2] = (src[1]>>1)-src[3]; add \arg7\().8h, \arg1\().8h, \arg7\().8h //int16 e[i][3] = src[1] + (src[3]>>1); // } .endm .macro TRANSFORM_TOTAL_16BITS arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // both row & col transform used // { // output: f_q[0]~[3], input: e_q[0]~[3]; add \arg0\().8h, \arg4\().8h, \arg7\().8h //int16 f[i][0] = e[i][0] + e[i][3]; add \arg1\().8h, \arg5\().8h, \arg6\().8h //int16 f[i][1] = e[i][1] + e[i][2]; sub \arg2\().8h, \arg5\().8h, \arg6\().8h //int16 f[i][2] = e[i][1] - e[i][2]; sub \arg3\().8h, \arg4\().8h, \arg7\().8h //int16 f[i][3] = e[i][0] - e[i][3]; // } .endm .macro ROW_TRANSFORM_0_STEP arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // { // input: src_d[0]~[3], output: e_q[0]~[3]; saddl \arg4\().4s, \arg0\().4h, \arg2\().4h //int32 e[i][0] = src[0] + src[2]; ssubl \arg5\().4s, \arg0\().4h, \arg2\().4h //int32 e[i][1] = src[0] - src[2]; ssubl \arg6\().4s, \arg1\().4h, \arg3\().4h //int32 e[i][2] = src[1] - src[3]; saddl \arg7\().4s, \arg1\().4h, \arg3\().4h //int32 e[i][3] = src[1] + src[3]; // } .endm .macro COL_TRANSFORM_0_STEP arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // { // input: src_q[0]~[3], output: e_q[0]~[3]; add \arg4\().4s, \arg0\().4s, \arg2\().4s //int32 e[0][j] = f[0][j] + f[2][j]; sub \arg5\().4s, \arg0\().4s, \arg2\().4s //int32 e[1][j] = f[0][j] - f[2][j]; sub \arg6\().4s, \arg1\().4s, \arg3\().4s //int32 e[2][j] = (f[1][j]>>1) - f[3][j]; add \arg7\().4s, \arg1\().4s, \arg3\().4s //int32 e[3][j] = f[1][j] + (f[3][j]>>1); // } .endm .macro TRANSFORM_4BYTES arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 // both row & col transform used // { // output: f_q[0]~[3], input: e_q[0]~[3]; add \arg0\().4s, \arg4\().4s, \arg7\().4s //int16 f[i][0] = e[i][0] + e[i][3]; add \arg1\().4s, \arg5\().4s, \arg6\().4s //int16 f[i][1] = e[i][1] + e[i][2]; sub \arg2\().4s, \arg5\().4s, \arg6\().4s //int16 f[i][2] = e[i][1] - e[i][2]; sub \arg3\().4s, \arg4\().4s, \arg7\().4s //int16 f[i][3] = e[i][0] - e[i][3]; // } .endm .macro MB_PRED_8BITS_ADD_DCT_16BITS_CLIP arg0, arg1, arg2, arg3, arg4 // { // input: pred_d[0](output), dct_q0/1, working_q0/1; uxtl \arg3\().8h, \arg0\().8b uxtl2 \arg4\().8h, \arg0\().16b add \arg3\().8h, \arg3\().8h, \arg1\().8h add \arg4\().8h, \arg4\().8h, \arg2\().8h sqxtun \arg0\().8b, \arg3\().8h sqxtun2 \arg0\().16b,\arg4\().8h // } .endm WELS_ASM_AARCH64_FUNC_BEGIN WelsGetNoneZeroCount_AArch64_neon ld1 {v0.8h, v1.8h}, [x0] ZERO_COUNT_IN_2_QUARWORD v0, v1, b0 mov x0, v0.d[0] mov x1, #16 subs x0, x1, x0 WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsQuant4x4_AArch64_neon ld1 {v2.8h}, [x1] ld1 {v0.8h, v1.8h}, [x0] ld1 {v3.8h}, [x2] mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS v0, v2, v3, v5, v6, v7 st1 {v2.8h}, [x0], #16 NEWQUANT_COEF_EACH_16BITS v1, v4, v3, v5, v6, v7 st1 {v4.8h}, [x0], #16 WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsQuant4x4Dc_AArch64_neon ld1 {v0.8h, v1.8h}, [x0] dup v2.8h, w1 // even ff range [0, 768] dup v3.8h, w2 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS v0, v2, v3, v5, v6, v7 st1 {v2.8h}, [x0], #16 NEWQUANT_COEF_EACH_16BITS v1, v4, v3, v5, v6, v7 st1 {v4.8h}, [x0], #16 WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsQuantFour4x4_AArch64_neon ld1 {v2.8h}, [x1] ld1 {v3.8h}, [x2] mov x1, x0 .rept 4 ld1 {v0.8h, v1.8h}, [x0], #32 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS v0, v4, v3, v5, v6, v7 st1 {v4.8h}, [x1], #16 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS v1, v4, v3, v5, v6, v7 st1 {v4.8h}, [x1], #16 .endr WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsQuantFour4x4Max_AArch64_neon ld1 {v2.8h}, [x1] ld1 {v3.8h}, [x2] mov x1, x0 ld1 {v0.8h, v1.8h}, [x0], #32 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v0, v4, v3, v5, v6, v7, v16 st1 {v4.8h}, [x1], #16 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v1, v4, v3, v5, v6, v7, v17 st1 {v4.8h}, [x1], #16 // then 1st 16 elem in v16 & v17 ld1 {v0.8h, v1.8h}, [x0], #32 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v0, v4, v3, v5, v6, v7, v18 st1 {v4.8h}, [x1], #16 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v1, v4, v3, v5, v6, v7, v19 st1 {v4.8h}, [x1], #16 // then 2st 16 elem in v18 & v19 SELECT_MAX_IN_ABS_COEF v16, v17, v18, v19, h20, h21 ld1 {v0.8h, v1.8h}, [x0], #32 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v0, v4, v3, v5, v6, v7, v16 st1 {v4.8h}, [x1], #16 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v1, v4, v3, v5, v6, v7, v17 st1 {v4.8h}, [x1], #16 // then 1st 16 elem in v16 & v17 ld1 {v0.8h, v1.8h}, [x0], #32 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v0, v4, v3, v5, v6, v7, v18 st1 {v4.8h}, [x1], #16 mov v4.16b, v2.16b NEWQUANT_COEF_EACH_16BITS_MAX v1, v4, v3, v5, v6, v7, v19 st1 {v4.8h}, [x1], #16 // then 2st 16 elem in v18 & v19 SELECT_MAX_IN_ABS_COEF v16, v17, v18, v19, h22, h23 st4 {v20.h,v21.h,v22.h,v23.h}[0], [x3] WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsDequant4x4_AArch64_neon ld1 {v0.8h, v1.8h}, [x0] ld1 {v2.8h}, [x1] mul v3.8h, v0.8h, v2.8h mul v4.8h, v1.8h, v2.8h st1 {v3.8h, v4.8h}, [x0] WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsDequantFour4x4_AArch64_neon ld1 {v2.8h}, [x1] mov x1, x0 .rept 4 ld1 {v0.8h,v1.8h}, [x0], #32 mul v3.8h, v0.8h, v2.8h mul v4.8h, v1.8h, v2.8h st1 {v3.8h,v4.8h}, [x1], #32 .endr WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsHadamardQuant2x2SkipKernel_AArch64_neon dup v4.8h, w1 mov x1, #32 ld1 {v0.h}[0], [x0], x1 //rs[0] ld1 {v0.h}[1], [x0], x1 //rs[16] ld1 {v0.h}[2], [x0], x1 //rs[32] ld1 {v0.h}[3], [x0], x1 //rs[48] HDM_QUANT_2x2_TOTAL_16BITS v0, v1, v2 // output v1 HDM_QUANT_2x2_TOTAL_16BITS v1, v0, v2 // output v0 abs v1.4h, v0.4h cmhi v0.4h, v1.4h, v4.4h // abs(dct[i])>threshold; mov w0, v0.s[0] mov w1, v0.s[1] orr w0, w0, w1 WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsHadamardQuant2x2_AArch64_neon dup v1.8h, w1 //ff dup v2.8h, w2 //mf eor v3.16b, v3.16b, v3.16b mov x1, #32 mov x2, x0 ld1 {v0.h}[0], [x0], x1 //rs[0] st1 {v3.h}[0], [x2], x1 //rs[00]=0 ld1 {v0.h}[1], [x0], x1 //rs[16] st1 {v3.h}[1], [x2], x1 //rs[16]=0 ld1 {v0.h}[2], [x0], x1 //rs[32] st1 {v3.h}[2], [x2], x1 //rs[32]=0 ld1 {v0.h}[3], [x0], x1 //rs[48] st1 {v3.h}[3], [x2], x1 //rs[48]=0 HDM_QUANT_2x2_TOTAL_16BITS v0, v4, v5 // output v4 HDM_QUANT_2x2_TOTAL_16BITS v4, v0, v5 // output v0 QUANT_DUALWORD_COEF_EACH_16BITS v0, v1, v2, v3, v4 st1 {v1.d}[0], [x3] // store to dct st1 {v1.d}[0], [x4] // store to block movi v3.8h, #1, lsl #0 movi v0.16b, #255 DC_ZERO_COUNT_IN_DUALWORD v1, h0, v3 mov x0, v0.d[0] mov x1, #4 subs x0, x1, x0 WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsDequantIHadamard4x4_AArch64_neon ld1 {v0.8h, v1.8h}, [x0] dup v4.8h, w1 IHDM_4x4_TOTAL_16BITS v0, v2, v3 IHDM_4x4_TOTAL_16BITS v1, v2, v3 MATRIX_TRANSFORM_EACH_16BITS_2x8_OUT2 v0, v1, v2, v3 IHDM_4x4_TOTAL_16BITS v0, v2, v3 mul v0.8h, v0.8h, v4.8h IHDM_4x4_TOTAL_16BITS v1, v2, v3 mul v1.8h, v1.8h, v4.8h MATRIX_TRANSFORM_EACH_16BITS_2x8_OUT2 v0, v1, v2, v3 st1 {v0.16b, v1.16b}, [x0] WELS_ASM_AARCH64_FUNC_END //void WelsDctT4_AArch64_neon (int16_t* pDct, uint8_t* pPixel1, int32_t iStride1, uint8_t* pPixel2, int32_t iStride2); WELS_ASM_AARCH64_FUNC_BEGIN WelsDctT4_AArch64_neon SIGN_EXTENSION x2, w2 SIGN_EXTENSION x4, w4 LOAD_4x4_DATA_FOR_DCT v0, v1, x1, x2, x3, x4 usubl v2.8h, v0.8b, v1.8b usubl2 v4.8h, v0.16b, v1.16b uzp1 v3.8h, v2.8h, v4.8h uzp2 v5.8h, v2.8h, v4.8h uzp2 v2.8h, v3.8h, v5.8h // s[2, 6, 10, 14] [3, 7, 11, 15] uzp1 v0.8h, v3.8h, v5.8h // s[0, 4, 8, 12] [1, 5, 9, 13] mov v3.d[0], v2.d[1] // s[3, 7, 11, 15] mov v1.d[0], v0.d[1] // s[1, 5, 9, 13] // horizontal transform DCT_ROW_TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 // transform element MATRIX_TRANSFORM_EACH_16BITS_OUT4 v0, v1, v2, v3, v4, v5, v6, v7 // vertical transform DCT_ROW_TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0] WELS_ASM_AARCH64_FUNC_END //void WelsDctFourT4_AArch64_neon (int16_t* pDct, uint8_t* pPixel1, int32_t iStride1, uint8_t* pPixel2, int32_t iStride2); WELS_ASM_AARCH64_FUNC_BEGIN WelsDctFourT4_AArch64_neon SIGN_EXTENSION x2,w2 SIGN_EXTENSION x4,w4 .rept 2 LOAD_8x4_DATA_FOR_DCT v0, v1, v2, v3, v4, v5, v6, v7, x1, x3 usubl v0.8h, v0.8b, v4.8b usubl v1.8h, v1.8b, v5.8b usubl v2.8h, v2.8b, v6.8b usubl v3.8h, v3.8b, v7.8b MATRIX_TRANSFORM_EACH_16BITS_OUT4 v0, v1, v2, v3, v4, v5, v6, v7 // horizontal transform DCT_ROW_TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 // transform element MATRIX_TRANSFORM_EACH_16BITS_OUT4 v0, v1, v2, v3, v4, v5, v6, v7 // vertical transform DCT_ROW_TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 uzp1 v4.2d, v0.2d, v1.2d uzp2 v6.2d, v0.2d, v1.2d uzp1 v5.2d, v2.2d, v3.2d uzp2 v7.2d, v2.2d, v3.2d st1 {v4.16b, v5.16b}, [x0], #32 st1 {v6.16b, v7.16b}, [x0], #32 .endr WELS_ASM_AARCH64_FUNC_END //void WelsIDctT4Rec_AArch64_neon (uint8_t* pRec, int32_t iStride, uint8_t* pPrediction, int32_t iPredStride, int16_t* pDct) WELS_ASM_AARCH64_FUNC_BEGIN WelsIDctT4Rec_AArch64_neon SIGN_EXTENSION x1,w1 SIGN_EXTENSION x3,w3 ld1 {v16.s}[0], [x2], x3 ld1 {v16.s}[1], [x2], x3 ld1 {v16.s}[2], [x2], x3 ld1 {v16.s}[3], [x2], x3 // Pred ld4 {v0.4h, v1.4h, v2.4h, v3.4h}, [x4] // dct coeff ROW_TRANSFORM_1_STEP_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 MATRIX_TRANSFORM_EACH_16BITS_OUT4 v0, v1, v2, v3, v4, v5, v6, v7 ROW_TRANSFORM_1_STEP_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 ins v0.d[1], v1.d[0] ins v2.d[1], v3.d[0] srshr v0.8h, v0.8h, #6 srshr v2.8h, v2.8h, #6 //after rounding 6, clip into [0, 255] uxtl v1.8h, v16.8b add v0.8h, v0.8h, v1.8h sqxtun v1.8b, v0.8h st1 {v1.s}[0],[x0],x1 st1 {v1.s}[1],[x0],x1 uxtl2 v1.8h, v16.16b add v2.8h, v2.8h, v1.8h sqxtun v1.8b, v2.8h st1 {v1.s}[0],[x0],x1 st1 {v1.s}[1],[x0],x1 WELS_ASM_AARCH64_FUNC_END //void WelsIDctFourT4Rec_AArch64_neon (uint8_t* pRec, int32_t iStride, uint8_t* pPrediction, int32_t iPredStride, int16_t* pDct); WELS_ASM_AARCH64_FUNC_BEGIN WelsIDctFourT4Rec_AArch64_neon SIGN_EXTENSION x1,w1 SIGN_EXTENSION x3,w3 .rept 2 ld1 {v16.d}[0], [x2], x3 ld1 {v16.d}[1], [x2], x3 ld1 {v17.d}[0], [x2], x3 ld1 {v17.d}[1], [x2], x3 // Pred ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [x4], #64 // dct coeff ROW_TRANSFORM_1_STEP_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 MATRIX_TRANSFORM_EACH_16BITS_OUT4 v0, v1, v2, v3, v4, v5, v6, v7 ROW_TRANSFORM_1_STEP_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 TRANSFORM_TOTAL_16BITS v0, v1, v2, v3, v4, v5, v6, v7 srshr v0.8h, v0.8h, #6 srshr v1.8h, v1.8h, #6 srshr v2.8h, v2.8h, #6 srshr v3.8h, v3.8h, #6 //after rounding 6, clip into [0, 255] uxtl v4.8h, v16.8b add v0.8h, v0.8h, v4.8h sqxtun v0.8b, v0.8h st1 {v0.d}[0],[x0],x1 uxtl2 v5.8h, v16.16b add v1.8h, v1.8h, v5.8h sqxtun v1.8b, v1.8h st1 {v1.d}[0],[x0],x1 uxtl v6.8h, v17.8b add v2.8h, v2.8h, v6.8h sqxtun v2.8b, v2.8h st1 {v2.d}[0],[x0],x1 uxtl2 v7.8h, v17.16b add v3.8h, v3.8h, v7.8h sqxtun v3.8b, v3.8h st1 {v3.d}[0],[x0],x1 .endr WELS_ASM_AARCH64_FUNC_END WELS_ASM_AARCH64_FUNC_BEGIN WelsHadamardT4Dc_AArch64_neon mov x2, #32 ld1 {v0.h}[0], [x1], x2 ld1 {v1.h}[0], [x1], x2 ld1 {v0.h}[1], [x1], x2 ld1 {v1.h}[1], [x1], x2 ld1 {v2.h}[0], [x1], x2 ld1 {v3.h}[0], [x1], x2 ld1 {v2.h}[1], [x1], x2 ld1 {v3.h}[1], [x1], x2 ld1 {v0.h}[2], [x1], x2 ld1 {v1.h}[2], [x1], x2 ld1 {v0.h}[3], [x1], x2 ld1 {v1.h}[3], [x1], x2 ld1 {v2.h}[2], [x1], x2 ld1 {v3.h}[2], [x1], x2 ld1 {v2.h}[3], [x1], x2 ld1 {v3.h}[3], [x1], x2 // v0[0 4 08 12],v1[1 5 09 13],v2[2 6 10 14],v3[3 7 11 15] ROW_TRANSFORM_0_STEP v0, v1, v3, v2, v4, v7, v6, v5 TRANSFORM_4BYTES v0, v1, v3, v2, v4, v7, v6, v5 // transform element 32bits uzp1 v4.4s, v0.4s, v1.4s // 0 2 4 6 uzp2 v5.4s, v0.4s, v1.4s // 1 3 5 7 uzp1 v6.4s, v2.4s, v3.4s // 8 10 12 14 uzp2 v7.4s, v2.4s, v3.4s // 9 11 13 15 uzp1 v0.4s, v4.4s, v6.4s // 0 4 8 12 uzp2 v2.4s, v4.4s, v6.4s // 2 6 10 14 uzp1 v1.4s, v5.4s, v7.4s // 1 5 9 13 uzp2 v3.4s, v5.4s, v7.4s // 3 7 11 15 COL_TRANSFORM_0_STEP v0, v1, v3, v2, v4, v7, v6, v5 TRANSFORM_4BYTES v0, v1, v3, v2, v4, v7, v6, v5 sqrshrn v4.4h, v0.4s, #1 sqrshrn2 v4.8h, v1.4s, #1 sqrshrn v5.4h, v2.4s, #1 sqrshrn2 v5.8h, v3.4s, #1 st1 {v4.16b, v5.16b}, [x0] //store WELS_ASM_AARCH64_FUNC_END //void WelsIDctRecI16x16Dc_AArch64_neon (uint8_t* pRec, int32_t iStride, uint8_t* pPrediction, int32_t iPredStride, // int16_t* pDctDc); WELS_ASM_AARCH64_FUNC_BEGIN WelsIDctRecI16x16Dc_AArch64_neon SIGN_EXTENSION x1,w1 SIGN_EXTENSION x3,w3 ld1 {v16.16b,v17.16b}, [x4] srshr v16.8h, v16.8h, #6 srshr v17.8h, v17.8h, #6 dup v0.8h, v16.h[0] dup v1.8h, v16.h[1] ins v0.d[1], v1.d[0] dup v1.8h, v16.h[2] dup v2.8h, v16.h[3] ins v1.d[1], v2.d[0] .rept 4 ld1 {v3.16b}, [x2], x3 MB_PRED_8BITS_ADD_DCT_16BITS_CLIP v3, v0, v1, v4, v5 st1 {v3.16b}, [x0], x1 .endr dup v0.8h, v16.h[4] dup v1.8h, v16.h[5] ins v0.d[1], v1.d[0] dup v1.8h, v16.h[6] dup v2.8h, v16.h[7] ins v1.d[1], v2.d[0] .rept 4 ld1 {v3.16b}, [x2], x3 MB_PRED_8BITS_ADD_DCT_16BITS_CLIP v3, v0, v1, v4, v5 st1 {v3.16b}, [x0], x1 .endr dup v0.8h, v17.h[0] dup v1.8h, v17.h[1] ins v0.d[1], v1.d[0] dup v1.8h, v17.h[2] dup v2.8h, v17.h[3] ins v1.d[1], v2.d[0] .rept 4 ld1 {v3.16b}, [x2], x3 MB_PRED_8BITS_ADD_DCT_16BITS_CLIP v3, v0, v1, v4, v5 st1 {v3.16b}, [x0], x1 .endr dup v0.8h, v17.h[4] dup v1.8h, v17.h[5] ins v0.d[1], v1.d[0] dup v1.8h, v17.h[6] dup v2.8h, v17.h[7] ins v1.d[1], v2.d[0] .rept 4 ld1 {v3.16b}, [x2], x3 MB_PRED_8BITS_ADD_DCT_16BITS_CLIP v3, v0, v1, v4, v5 st1 {v3.16b}, [x0], x1 .endr WELS_ASM_AARCH64_FUNC_END #endif