ref: cf35ffc025086588db5ccab977a94a325a50f949
dir: /vp9/encoder/x86/vp9_highbd_error_avx.asm/
; ; Copyright (c) 2015 The WebM project authors. All Rights Reserved. ; ; Use of this source code is governed by a BSD-style license ; that can be found in the LICENSE file in the root of the source ; tree. An additional intellectual property rights grant can be found ; in the file PATENTS. All contributing project authors may ; be found in the AUTHORS file in the root of the source tree. ; %define private_prefix vp9 %include "third_party/x86inc/x86inc.asm" SECTION .text ALIGN 16 ; ; int64_t vp9_highbd_block_error_8bit(int32_t *coeff, int32_t *dqcoeff, ; intptr_t block_size, int64_t *ssz) ; INIT_XMM avx cglobal highbd_block_error_8bit, 4, 5, 8, uqc, dqc, size, ssz vzeroupper ; If only one iteration is required, then handle this as a special case. ; It is the most frequent case, so we can have a significant gain here ; by not setting up a loop and accumulators. cmp sizeq, 16 jne .generic ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Common case of size == 16 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Load input vectors mova xm0, [dqcq] packssdw xm0, [dqcq+16] mova xm2, [uqcq] packssdw xm2, [uqcq+16] mova xm1, [dqcq+32] packssdw xm1, [dqcq+48] mova xm3, [uqcq+32] packssdw xm3, [uqcq+48] ; Compute the errors. psubw xm0, xm2 psubw xm1, xm3 ; Individual errors are max 15bit+sign, so squares are 30bit, and ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit). pmaddwd xm2, xm2 pmaddwd xm3, xm3 pmaddwd xm0, xm0 pmaddwd xm1, xm1 ; Squares are always positive, so we can use unsigned arithmetic after ; squaring. As mentioned earlier 2 sums fit in 31 bits, so 4 sums will ; fit in 32bits paddd xm2, xm3 paddd xm0, xm1 ; Accumulate horizontally in 64 bits, there is no chance of overflow here pxor xm5, xm5 pblendw xm3, xm5, xm2, 0x33 ; Zero extended low of a pair of 32 bits psrlq xm2, 32 ; Zero extended high of a pair of 32 bits pblendw xm1, xm5, xm0, 0x33 ; Zero extended low of a pair of 32 bits psrlq xm0, 32 ; Zero extended high of a pair of 32 bits paddq xm2, xm3 paddq xm0, xm1 psrldq xm3, xm2, 8 psrldq xm1, xm0, 8 paddq xm2, xm3 paddq xm0, xm1 ; Store the return value %if ARCH_X86_64 movq rax, xm0 movq [sszq], xm2 %else movd eax, xm0 pextrd edx, xm0, 1 movq [sszd], xm2 %endif RET ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Generic case of size != 16, speculative low precision ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ALIGN 16 .generic: pxor xm4, xm4 ; sse accumulator pxor xm5, xm5 ; overflow detection register for xm4 pxor xm6, xm6 ; ssz accumulator pxor xm7, xm7 ; overflow detection register for xm6 lea uqcq, [uqcq+sizeq*4] lea dqcq, [dqcq+sizeq*4] neg sizeq ; Push the negative size as the high precision code might need it push sizeq .loop: ; Load input vectors mova xm0, [dqcq+sizeq*4] packssdw xm0, [dqcq+sizeq*4+16] mova xm2, [uqcq+sizeq*4] packssdw xm2, [uqcq+sizeq*4+16] mova xm1, [dqcq+sizeq*4+32] packssdw xm1, [dqcq+sizeq*4+48] mova xm3, [uqcq+sizeq*4+32] packssdw xm3, [uqcq+sizeq*4+48] add sizeq, 16 ; Compute the squared errors. ; Individual errors are max 15bit+sign, so squares are 30bit, and ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit). psubw xm0, xm2 pmaddwd xm2, xm2 pmaddwd xm0, xm0 psubw xm1, xm3 pmaddwd xm3, xm3 pmaddwd xm1, xm1 ; Squares are always positive, so we can use unsigned arithmetic after ; squaring. As mentioned earlier 2 sums fit in 31 bits, so 4 sums will ; fit in 32bits paddd xm2, xm3 paddd xm0, xm1 ; We accumulate using 32 bit arithmetic, but detect potential overflow ; by checking if the MSB of the accumulators have ever been a set bit. ; If yes, we redo the whole compute at the end on higher precision, but ; this happens extremely rarely, so we still achieve a net gain. paddd xm4, xm0 paddd xm6, xm2 por xm5, xm4 ; OR in the accumulator for overflow detection por xm7, xm6 ; OR in the accumulator for overflow detection jnz .loop ; Add pairs horizontally (still only on 32 bits) phaddd xm4, xm4 por xm5, xm4 ; OR in the accumulator for overflow detection phaddd xm6, xm6 por xm7, xm6 ; OR in the accumulator for overflow detection ; Check for possibility of overflow by testing if bit 32 of each dword lane ; have ever been set. If they were not, then there was no overflow and the ; final sum will fit in 32 bits. If overflow happened, then ; we redo the whole computation on higher precision. por xm7, xm5 pmovmskb r4, xm7 test r4, 0x8888 jnz .highprec phaddd xm4, xm4 phaddd xm6, xm6 pmovzxdq xm4, xm4 pmovzxdq xm6, xm6 ; Restore stack pop sizeq ; Store the return value %if ARCH_X86_64 movq rax, xm4 movq [sszq], xm6 %else movd eax, xm4 pextrd edx, xm4, 1 movq [sszd], xm6 %endif RET ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Generic case of size != 16, high precision case ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; .highprec: pxor xm4, xm4 ; sse accumulator pxor xm5, xm5 ; dedicated zero register pxor xm6, xm6 ; ssz accumulator pop sizeq .loophp: mova xm0, [dqcq+sizeq*4] packssdw xm0, [dqcq+sizeq*4+16] mova xm2, [uqcq+sizeq*4] packssdw xm2, [uqcq+sizeq*4+16] mova xm1, [dqcq+sizeq*4+32] packssdw xm1, [dqcq+sizeq*4+48] mova xm3, [uqcq+sizeq*4+32] packssdw xm3, [uqcq+sizeq*4+48] add sizeq, 16 ; individual errors are max. 15bit+sign, so squares are 30bit, and ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit) psubw xm0, xm2 pmaddwd xm2, xm2 pmaddwd xm0, xm0 psubw xm1, xm3 pmaddwd xm3, xm3 pmaddwd xm1, xm1 ; accumulate in 64bit punpckldq xm7, xm0, xm5 punpckhdq xm0, xm5 paddq xm4, xm7 punpckldq xm7, xm2, xm5 punpckhdq xm2, xm5 paddq xm6, xm7 punpckldq xm7, xm1, xm5 punpckhdq xm1, xm5 paddq xm4, xm7 punpckldq xm7, xm3, xm5 punpckhdq xm3, xm5 paddq xm6, xm7 paddq xm4, xm0 paddq xm4, xm1 paddq xm6, xm2 paddq xm6, xm3 jnz .loophp ; Accumulate horizontally movhlps xm5, xm4 movhlps xm7, xm6 paddq xm4, xm5 paddq xm6, xm7 ; Store the return value %if ARCH_X86_64 movq rax, xm4 movq [sszq], xm6 %else movd eax, xm4 pextrd edx, xm4, 1 movq [sszd], xm6 %endif RET END