| ; |
| ; Copyright (c) 2015 The WebM project authors. All Rights Reserved. |
| ; |
| ; Use of this source code is governed by a BSD-style license |
| ; that can be found in the LICENSE file in the root of the source |
| ; tree. An additional intellectual property rights grant can be found |
| ; in the file PATENTS. All contributing project authors may |
| ; be found in the AUTHORS file in the root of the source tree. |
| ; |
| |
| %define private_prefix vp9 |
| |
| %include "third_party/x86inc/x86inc.asm" |
| |
| SECTION .text |
| ALIGN 16 |
| |
| ; |
| ; int64_t vp9_highbd_block_error_8bit(int32_t *coeff, int32_t *dqcoeff, |
| ; intptr_t block_size, int64_t *ssz) |
| ; |
| |
| INIT_XMM avx |
| cglobal highbd_block_error_8bit, 4, 5, 8, uqc, dqc, size, ssz |
| vzeroupper |
| |
| ; If only one iteration is required, then handle this as a special case. |
| ; It is the most frequent case, so we can have a significant gain here |
| ; by not setting up a loop and accumulators. |
| cmp sizeq, 16 |
| jne .generic |
| |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ;; Common case of size == 16 |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| |
| ; Load input vectors |
| mova xm0, [dqcq] |
| packssdw xm0, [dqcq+16] |
| mova xm2, [uqcq] |
| packssdw xm2, [uqcq+16] |
| |
| mova xm1, [dqcq+32] |
| packssdw xm1, [dqcq+48] |
| mova xm3, [uqcq+32] |
| packssdw xm3, [uqcq+48] |
| |
| ; Compute the errors. |
| psubw xm0, xm2 |
| psubw xm1, xm3 |
| |
| ; Individual errors are max 15bit+sign, so squares are 30bit, and |
| ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit). |
| pmaddwd xm2, xm2 |
| pmaddwd xm3, xm3 |
| |
| pmaddwd xm0, xm0 |
| pmaddwd xm1, xm1 |
| |
| ; Squares are always positive, so we can use unsigned arithmetic after |
| ; squaring. As mentioned earlier 2 sums fit in 31 bits, so 4 sums will |
| ; fit in 32bits |
| paddd xm2, xm3 |
| paddd xm0, xm1 |
| |
| ; Accumulate horizontally in 64 bits, there is no chance of overflow here |
| pxor xm5, xm5 |
| |
| pblendw xm3, xm5, xm2, 0x33 ; Zero extended low of a pair of 32 bits |
| psrlq xm2, 32 ; Zero extended high of a pair of 32 bits |
| |
| pblendw xm1, xm5, xm0, 0x33 ; Zero extended low of a pair of 32 bits |
| psrlq xm0, 32 ; Zero extended high of a pair of 32 bits |
| |
| paddq xm2, xm3 |
| paddq xm0, xm1 |
| |
| psrldq xm3, xm2, 8 |
| psrldq xm1, xm0, 8 |
| |
| paddq xm2, xm3 |
| paddq xm0, xm1 |
| |
| ; Store the return value |
| %if ARCH_X86_64 |
| movq rax, xm0 |
| movq [sszq], xm2 |
| %else |
| movd eax, xm0 |
| pextrd edx, xm0, 1 |
| movq [sszd], xm2 |
| %endif |
| RET |
| |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ;; Generic case of size != 16, speculative low precision |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ALIGN 16 |
| .generic: |
| pxor xm4, xm4 ; sse accumulator |
| pxor xm5, xm5 ; overflow detection register for xm4 |
| pxor xm6, xm6 ; ssz accumulator |
| pxor xm7, xm7 ; overflow detection register for xm6 |
| lea uqcq, [uqcq+sizeq*4] |
| lea dqcq, [dqcq+sizeq*4] |
| neg sizeq |
| |
| ; Push the negative size as the high precision code might need it |
| push sizeq |
| |
| .loop: |
| ; Load input vectors |
| mova xm0, [dqcq+sizeq*4] |
| packssdw xm0, [dqcq+sizeq*4+16] |
| mova xm2, [uqcq+sizeq*4] |
| packssdw xm2, [uqcq+sizeq*4+16] |
| |
| mova xm1, [dqcq+sizeq*4+32] |
| packssdw xm1, [dqcq+sizeq*4+48] |
| mova xm3, [uqcq+sizeq*4+32] |
| packssdw xm3, [uqcq+sizeq*4+48] |
| |
| add sizeq, 16 |
| |
| ; Compute the squared errors. |
| ; Individual errors are max 15bit+sign, so squares are 30bit, and |
| ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit). |
| psubw xm0, xm2 |
| pmaddwd xm2, xm2 |
| pmaddwd xm0, xm0 |
| |
| psubw xm1, xm3 |
| pmaddwd xm3, xm3 |
| pmaddwd xm1, xm1 |
| |
| ; Squares are always positive, so we can use unsigned arithmetic after |
| ; squaring. As mentioned earlier 2 sums fit in 31 bits, so 4 sums will |
| ; fit in 32bits |
| paddd xm2, xm3 |
| paddd xm0, xm1 |
| |
| ; We accumulate using 32 bit arithmetic, but detect potential overflow |
| ; by checking if the MSB of the accumulators have ever been a set bit. |
| ; If yes, we redo the whole compute at the end on higher precision, but |
| ; this happens extremely rarely, so we still achieve a net gain. |
| paddd xm4, xm0 |
| paddd xm6, xm2 |
| por xm5, xm4 ; OR in the accumulator for overflow detection |
| por xm7, xm6 ; OR in the accumulator for overflow detection |
| |
| jnz .loop |
| |
| ; Add pairs horizontally (still only on 32 bits) |
| phaddd xm4, xm4 |
| por xm5, xm4 ; OR in the accumulator for overflow detection |
| phaddd xm6, xm6 |
| por xm7, xm6 ; OR in the accumulator for overflow detection |
| |
| ; Check for possibility of overflow by testing if bit 32 of each dword lane |
| ; have ever been set. If they were not, then there was no overflow and the |
| ; final sum will fit in 32 bits. If overflow happened, then |
| ; we redo the whole computation on higher precision. |
| por xm7, xm5 |
| pmovmskb r4, xm7 |
| test r4, 0x8888 |
| jnz .highprec |
| |
| phaddd xm4, xm4 |
| phaddd xm6, xm6 |
| pmovzxdq xm4, xm4 |
| pmovzxdq xm6, xm6 |
| |
| ; Restore stack |
| pop sizeq |
| |
| ; Store the return value |
| %if ARCH_X86_64 |
| movq rax, xm4 |
| movq [sszq], xm6 |
| %else |
| movd eax, xm4 |
| pextrd edx, xm4, 1 |
| movq [sszd], xm6 |
| %endif |
| RET |
| |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ;; Generic case of size != 16, high precision case |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| .highprec: |
| pxor xm4, xm4 ; sse accumulator |
| pxor xm5, xm5 ; dedicated zero register |
| pxor xm6, xm6 ; ssz accumulator |
| pop sizeq |
| |
| .loophp: |
| mova xm0, [dqcq+sizeq*4] |
| packssdw xm0, [dqcq+sizeq*4+16] |
| mova xm2, [uqcq+sizeq*4] |
| packssdw xm2, [uqcq+sizeq*4+16] |
| |
| mova xm1, [dqcq+sizeq*4+32] |
| packssdw xm1, [dqcq+sizeq*4+48] |
| mova xm3, [uqcq+sizeq*4+32] |
| packssdw xm3, [uqcq+sizeq*4+48] |
| |
| add sizeq, 16 |
| |
| ; individual errors are max. 15bit+sign, so squares are 30bit, and |
| ; thus the sum of 2 should fit in a 31bit integer (+ unused sign bit) |
| |
| psubw xm0, xm2 |
| pmaddwd xm2, xm2 |
| pmaddwd xm0, xm0 |
| |
| psubw xm1, xm3 |
| pmaddwd xm3, xm3 |
| pmaddwd xm1, xm1 |
| |
| ; accumulate in 64bit |
| punpckldq xm7, xm0, xm5 |
| punpckhdq xm0, xm5 |
| paddq xm4, xm7 |
| |
| punpckldq xm7, xm2, xm5 |
| punpckhdq xm2, xm5 |
| paddq xm6, xm7 |
| |
| punpckldq xm7, xm1, xm5 |
| punpckhdq xm1, xm5 |
| paddq xm4, xm7 |
| |
| punpckldq xm7, xm3, xm5 |
| punpckhdq xm3, xm5 |
| paddq xm6, xm7 |
| |
| paddq xm4, xm0 |
| paddq xm4, xm1 |
| paddq xm6, xm2 |
| paddq xm6, xm3 |
| |
| jnz .loophp |
| |
| ; Accumulate horizontally |
| movhlps xm5, xm4 |
| movhlps xm7, xm6 |
| paddq xm4, xm5 |
| paddq xm6, xm7 |
| |
| ; Store the return value |
| %if ARCH_X86_64 |
| movq rax, xm4 |
| movq [sszq], xm6 |
| %else |
| movd eax, xm4 |
| pextrd edx, xm4, 1 |
| movq [sszd], xm6 |
| %endif |
| RET |
| |
| END |
