| ; |
| ; Copyright (c) 2010 The WebM project authors. All Rights Reserved. |
| ; |
| ; Use of this source code is governed by a BSD-style license |
| ; that can be found in the LICENSE file in the root of the source |
| ; tree. An additional intellectual property rights grant can be found |
| ; in the file PATENTS. All contributing project authors may |
| ; be found in the AUTHORS file in the root of the source tree. |
| ; |
| |
| |
| %include "vpx_ports/x86_abi_support.asm" |
| |
| %macro STACK_FRAME_CREATE 0 |
| %if ABI_IS_32BIT |
| %define input rsi |
| %define output rdi |
| %define pitch rax |
| push rbp |
| mov rbp, rsp |
| GET_GOT rbx |
| push rsi |
| push rdi |
| ; end prolog |
| |
| mov rsi, arg(0) |
| mov rdi, arg(1) |
| |
| movsxd rax, dword ptr arg(2) |
| lea rcx, [rsi + rax*2] |
| %else |
| %if LIBVPX_YASM_WIN64 |
| %define input rcx |
| %define output rdx |
| %define pitch r8 |
| SAVE_XMM 7, u |
| %else |
| %define input rdi |
| %define output rsi |
| %define pitch rdx |
| %endif |
| %endif |
| %endmacro |
| |
| %macro STACK_FRAME_DESTROY 0 |
| %define input |
| %define output |
| %define pitch |
| |
| %if ABI_IS_32BIT |
| pop rdi |
| pop rsi |
| RESTORE_GOT |
| pop rbp |
| %else |
| %if LIBVPX_YASM_WIN64 |
| RESTORE_XMM |
| %endif |
| %endif |
| ret |
| %endmacro |
| |
| ;void vp8_short_fdct4x4_sse2(short *input, short *output, int pitch) |
| global sym(vp8_short_fdct4x4_sse2) PRIVATE |
| sym(vp8_short_fdct4x4_sse2): |
| |
| STACK_FRAME_CREATE |
| |
| movq xmm0, MMWORD PTR[input ] ;03 02 01 00 |
| movq xmm2, MMWORD PTR[input+ pitch] ;13 12 11 10 |
| lea input, [input+2*pitch] |
| movq xmm1, MMWORD PTR[input ] ;23 22 21 20 |
| movq xmm3, MMWORD PTR[input+ pitch] ;33 32 31 30 |
| |
| punpcklqdq xmm0, xmm2 ;13 12 11 10 03 02 01 00 |
| punpcklqdq xmm1, xmm3 ;33 32 31 30 23 22 21 20 |
| |
| movdqa xmm2, xmm0 |
| punpckldq xmm0, xmm1 ;23 22 03 02 21 20 01 00 |
| punpckhdq xmm2, xmm1 ;33 32 13 12 31 30 11 10 |
| movdqa xmm1, xmm0 |
| punpckldq xmm0, xmm2 ;31 21 30 20 11 10 01 00 |
| pshufhw xmm1, xmm1, 0b1h ;22 23 02 03 xx xx xx xx |
| pshufhw xmm2, xmm2, 0b1h ;32 33 12 13 xx xx xx xx |
| |
| punpckhdq xmm1, xmm2 ;32 33 22 23 12 13 02 03 |
| movdqa xmm3, xmm0 |
| paddw xmm0, xmm1 ;b1 a1 b1 a1 b1 a1 b1 a1 |
| psubw xmm3, xmm1 ;c1 d1 c1 d1 c1 d1 c1 d1 |
| psllw xmm0, 3 ;b1 <<= 3 a1 <<= 3 |
| psllw xmm3, 3 ;c1 <<= 3 d1 <<= 3 |
| |
| movdqa xmm1, xmm0 |
| pmaddwd xmm0, XMMWORD PTR[GLOBAL(_mult_add)] ;a1 + b1 |
| pmaddwd xmm1, XMMWORD PTR[GLOBAL(_mult_sub)] ;a1 - b1 |
| movdqa xmm4, xmm3 |
| pmaddwd xmm3, XMMWORD PTR[GLOBAL(_5352_2217)] ;c1*2217 + d1*5352 |
| pmaddwd xmm4, XMMWORD PTR[GLOBAL(_2217_neg5352)];d1*2217 - c1*5352 |
| |
| paddd xmm3, XMMWORD PTR[GLOBAL(_14500)] |
| paddd xmm4, XMMWORD PTR[GLOBAL(_7500)] |
| psrad xmm3, 12 ;(c1 * 2217 + d1 * 5352 + 14500)>>12 |
| psrad xmm4, 12 ;(d1 * 2217 - c1 * 5352 + 7500)>>12 |
| |
| packssdw xmm0, xmm1 ;op[2] op[0] |
| packssdw xmm3, xmm4 ;op[3] op[1] |
| ; 23 22 21 20 03 02 01 00 |
| ; |
| ; 33 32 31 30 13 12 11 10 |
| ; |
| movdqa xmm2, xmm0 |
| punpcklqdq xmm0, xmm3 ;13 12 11 10 03 02 01 00 |
| punpckhqdq xmm2, xmm3 ;23 22 21 20 33 32 31 30 |
| |
| movdqa xmm3, xmm0 |
| punpcklwd xmm0, xmm2 ;32 30 22 20 12 10 02 00 |
| punpckhwd xmm3, xmm2 ;33 31 23 21 13 11 03 01 |
| movdqa xmm2, xmm0 |
| punpcklwd xmm0, xmm3 ;13 12 11 10 03 02 01 00 |
| punpckhwd xmm2, xmm3 ;33 32 31 30 23 22 21 20 |
| |
| movdqa xmm5, XMMWORD PTR[GLOBAL(_7)] |
| pshufd xmm2, xmm2, 04eh |
| movdqa xmm3, xmm0 |
| paddw xmm0, xmm2 ;b1 b1 b1 b1 a1 a1 a1 a1 |
| psubw xmm3, xmm2 ;c1 c1 c1 c1 d1 d1 d1 d1 |
| |
| pshufd xmm0, xmm0, 0d8h ;b1 b1 a1 a1 b1 b1 a1 a1 |
| movdqa xmm2, xmm3 ;save d1 for compare |
| pshufd xmm3, xmm3, 0d8h ;c1 c1 d1 d1 c1 c1 d1 d1 |
| pshuflw xmm0, xmm0, 0d8h ;b1 b1 a1 a1 b1 a1 b1 a1 |
| pshuflw xmm3, xmm3, 0d8h ;c1 c1 d1 d1 c1 d1 c1 d1 |
| pshufhw xmm0, xmm0, 0d8h ;b1 a1 b1 a1 b1 a1 b1 a1 |
| pshufhw xmm3, xmm3, 0d8h ;c1 d1 c1 d1 c1 d1 c1 d1 |
| movdqa xmm1, xmm0 |
| pmaddwd xmm0, XMMWORD PTR[GLOBAL(_mult_add)] ;a1 + b1 |
| pmaddwd xmm1, XMMWORD PTR[GLOBAL(_mult_sub)] ;a1 - b1 |
| |
| pxor xmm4, xmm4 ;zero out for compare |
| paddd xmm0, xmm5 |
| paddd xmm1, xmm5 |
| pcmpeqw xmm2, xmm4 |
| psrad xmm0, 4 ;(a1 + b1 + 7)>>4 |
| psrad xmm1, 4 ;(a1 - b1 + 7)>>4 |
| pandn xmm2, XMMWORD PTR[GLOBAL(_cmp_mask)] ;clear upper, |
| ;and keep bit 0 of lower |
| |
| movdqa xmm4, xmm3 |
| pmaddwd xmm3, XMMWORD PTR[GLOBAL(_5352_2217)] ;c1*2217 + d1*5352 |
| pmaddwd xmm4, XMMWORD PTR[GLOBAL(_2217_neg5352)] ;d1*2217 - c1*5352 |
| paddd xmm3, XMMWORD PTR[GLOBAL(_12000)] |
| paddd xmm4, XMMWORD PTR[GLOBAL(_51000)] |
| packssdw xmm0, xmm1 ;op[8] op[0] |
| psrad xmm3, 16 ;(c1 * 2217 + d1 * 5352 + 12000)>>16 |
| psrad xmm4, 16 ;(d1 * 2217 - c1 * 5352 + 51000)>>16 |
| |
| packssdw xmm3, xmm4 ;op[12] op[4] |
| movdqa xmm1, xmm0 |
| paddw xmm3, xmm2 ;op[4] += (d1!=0) |
| punpcklqdq xmm0, xmm3 ;op[4] op[0] |
| punpckhqdq xmm1, xmm3 ;op[12] op[8] |
| |
| movdqa XMMWORD PTR[output + 0], xmm0 |
| movdqa XMMWORD PTR[output + 16], xmm1 |
| |
| STACK_FRAME_DESTROY |
| |
| ;void vp8_short_fdct8x4_sse2(short *input, short *output, int pitch) |
| global sym(vp8_short_fdct8x4_sse2) PRIVATE |
| sym(vp8_short_fdct8x4_sse2): |
| |
| STACK_FRAME_CREATE |
| |
| ; read the input data |
| movdqa xmm0, [input ] |
| movdqa xmm2, [input+ pitch] |
| lea input, [input+2*pitch] |
| movdqa xmm4, [input ] |
| movdqa xmm3, [input+ pitch] |
| |
| ; transpose for the first stage |
| movdqa xmm1, xmm0 ; 00 01 02 03 04 05 06 07 |
| movdqa xmm5, xmm4 ; 20 21 22 23 24 25 26 27 |
| |
| punpcklwd xmm0, xmm2 ; 00 10 01 11 02 12 03 13 |
| punpckhwd xmm1, xmm2 ; 04 14 05 15 06 16 07 17 |
| |
| punpcklwd xmm4, xmm3 ; 20 30 21 31 22 32 23 33 |
| punpckhwd xmm5, xmm3 ; 24 34 25 35 26 36 27 37 |
| |
| movdqa xmm2, xmm0 ; 00 10 01 11 02 12 03 13 |
| punpckldq xmm0, xmm4 ; 00 10 20 30 01 11 21 31 |
| |
| punpckhdq xmm2, xmm4 ; 02 12 22 32 03 13 23 33 |
| |
| movdqa xmm4, xmm1 ; 04 14 05 15 06 16 07 17 |
| punpckldq xmm4, xmm5 ; 04 14 24 34 05 15 25 35 |
| |
| punpckhdq xmm1, xmm5 ; 06 16 26 36 07 17 27 37 |
| movdqa xmm3, xmm2 ; 02 12 22 32 03 13 23 33 |
| |
| punpckhqdq xmm3, xmm1 ; 03 13 23 33 07 17 27 37 |
| punpcklqdq xmm2, xmm1 ; 02 12 22 32 06 16 26 36 |
| |
| movdqa xmm1, xmm0 ; 00 10 20 30 01 11 21 31 |
| punpcklqdq xmm0, xmm4 ; 00 10 20 30 04 14 24 34 |
| |
| punpckhqdq xmm1, xmm4 ; 01 11 21 32 05 15 25 35 |
| |
| ; xmm0 0 |
| ; xmm1 1 |
| ; xmm2 2 |
| ; xmm3 3 |
| |
| ; first stage |
| movdqa xmm5, xmm0 |
| movdqa xmm4, xmm1 |
| |
| paddw xmm0, xmm3 ; a1 = 0 + 3 |
| paddw xmm1, xmm2 ; b1 = 1 + 2 |
| |
| psubw xmm4, xmm2 ; c1 = 1 - 2 |
| psubw xmm5, xmm3 ; d1 = 0 - 3 |
| |
| psllw xmm5, 3 |
| psllw xmm4, 3 |
| |
| psllw xmm0, 3 |
| psllw xmm1, 3 |
| |
| ; output 0 and 2 |
| movdqa xmm2, xmm0 ; a1 |
| |
| paddw xmm0, xmm1 ; op[0] = a1 + b1 |
| psubw xmm2, xmm1 ; op[2] = a1 - b1 |
| |
| ; output 1 and 3 |
| ; interleave c1, d1 |
| movdqa xmm1, xmm5 ; d1 |
| punpcklwd xmm1, xmm4 ; c1 d1 |
| punpckhwd xmm5, xmm4 ; c1 d1 |
| |
| movdqa xmm3, xmm1 |
| movdqa xmm4, xmm5 |
| |
| pmaddwd xmm1, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352 |
| pmaddwd xmm4, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352 |
| |
| pmaddwd xmm3, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352 |
| pmaddwd xmm5, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352 |
| |
| paddd xmm1, XMMWORD PTR[GLOBAL(_14500)] |
| paddd xmm4, XMMWORD PTR[GLOBAL(_14500)] |
| paddd xmm3, XMMWORD PTR[GLOBAL(_7500)] |
| paddd xmm5, XMMWORD PTR[GLOBAL(_7500)] |
| |
| psrad xmm1, 12 ; (c1 * 2217 + d1 * 5352 + 14500)>>12 |
| psrad xmm4, 12 ; (c1 * 2217 + d1 * 5352 + 14500)>>12 |
| psrad xmm3, 12 ; (d1 * 2217 - c1 * 5352 + 7500)>>12 |
| psrad xmm5, 12 ; (d1 * 2217 - c1 * 5352 + 7500)>>12 |
| |
| packssdw xmm1, xmm4 ; op[1] |
| packssdw xmm3, xmm5 ; op[3] |
| |
| ; done with vertical |
| ; transpose for the second stage |
| movdqa xmm4, xmm0 ; 00 10 20 30 04 14 24 34 |
| movdqa xmm5, xmm2 ; 02 12 22 32 06 16 26 36 |
| |
| punpcklwd xmm0, xmm1 ; 00 01 10 11 20 21 30 31 |
| punpckhwd xmm4, xmm1 ; 04 05 14 15 24 25 34 35 |
| |
| punpcklwd xmm2, xmm3 ; 02 03 12 13 22 23 32 33 |
| punpckhwd xmm5, xmm3 ; 06 07 16 17 26 27 36 37 |
| |
| movdqa xmm1, xmm0 ; 00 01 10 11 20 21 30 31 |
| punpckldq xmm0, xmm2 ; 00 01 02 03 10 11 12 13 |
| |
| punpckhdq xmm1, xmm2 ; 20 21 22 23 30 31 32 33 |
| |
| movdqa xmm2, xmm4 ; 04 05 14 15 24 25 34 35 |
| punpckldq xmm2, xmm5 ; 04 05 06 07 14 15 16 17 |
| |
| punpckhdq xmm4, xmm5 ; 24 25 26 27 34 35 36 37 |
| movdqa xmm3, xmm1 ; 20 21 22 23 30 31 32 33 |
| |
| punpckhqdq xmm3, xmm4 ; 30 31 32 33 34 35 36 37 |
| punpcklqdq xmm1, xmm4 ; 20 21 22 23 24 25 26 27 |
| |
| movdqa xmm4, xmm0 ; 00 01 02 03 10 11 12 13 |
| punpcklqdq xmm0, xmm2 ; 00 01 02 03 04 05 06 07 |
| |
| punpckhqdq xmm4, xmm2 ; 10 11 12 13 14 15 16 17 |
| |
| ; xmm0 0 |
| ; xmm1 4 |
| ; xmm2 1 |
| ; xmm3 3 |
| |
| movdqa xmm5, xmm0 |
| movdqa xmm2, xmm1 |
| |
| paddw xmm0, xmm3 ; a1 = 0 + 3 |
| paddw xmm1, xmm4 ; b1 = 1 + 2 |
| |
| psubw xmm4, xmm2 ; c1 = 1 - 2 |
| psubw xmm5, xmm3 ; d1 = 0 - 3 |
| |
| pxor xmm6, xmm6 ; zero out for compare |
| |
| pcmpeqw xmm6, xmm5 ; d1 != 0 |
| |
| pandn xmm6, XMMWORD PTR[GLOBAL(_cmp_mask8x4)] ; clear upper, |
| ; and keep bit 0 of lower |
| |
| ; output 0 and 2 |
| movdqa xmm2, xmm0 ; a1 |
| |
| paddw xmm0, xmm1 ; a1 + b1 |
| psubw xmm2, xmm1 ; a1 - b1 |
| |
| paddw xmm0, XMMWORD PTR[GLOBAL(_7w)] |
| paddw xmm2, XMMWORD PTR[GLOBAL(_7w)] |
| |
| psraw xmm0, 4 ; op[0] = (a1 + b1 + 7)>>4 |
| psraw xmm2, 4 ; op[8] = (a1 - b1 + 7)>>4 |
| |
| ; output 1 and 3 |
| ; interleave c1, d1 |
| movdqa xmm1, xmm5 ; d1 |
| punpcklwd xmm1, xmm4 ; c1 d1 |
| punpckhwd xmm5, xmm4 ; c1 d1 |
| |
| movdqa xmm3, xmm1 |
| movdqa xmm4, xmm5 |
| |
| pmaddwd xmm1, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352 |
| pmaddwd xmm4, XMMWORD PTR[GLOBAL (_5352_2217)] ; c1*2217 + d1*5352 |
| |
| pmaddwd xmm3, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352 |
| pmaddwd xmm5, XMMWORD PTR[GLOBAL(_2217_neg5352)] ; d1*2217 - c1*5352 |
| |
| paddd xmm1, XMMWORD PTR[GLOBAL(_12000)] |
| paddd xmm4, XMMWORD PTR[GLOBAL(_12000)] |
| paddd xmm3, XMMWORD PTR[GLOBAL(_51000)] |
| paddd xmm5, XMMWORD PTR[GLOBAL(_51000)] |
| |
| psrad xmm1, 16 ; (c1 * 2217 + d1 * 5352 + 14500)>>16 |
| psrad xmm4, 16 ; (c1 * 2217 + d1 * 5352 + 14500)>>16 |
| psrad xmm3, 16 ; (d1 * 2217 - c1 * 5352 + 7500)>>16 |
| psrad xmm5, 16 ; (d1 * 2217 - c1 * 5352 + 7500)>>16 |
| |
| packssdw xmm1, xmm4 ; op[4] |
| packssdw xmm3, xmm5 ; op[12] |
| |
| paddw xmm1, xmm6 ; op[4] += (d1!=0) |
| |
| movdqa xmm4, xmm0 |
| movdqa xmm5, xmm2 |
| |
| punpcklqdq xmm0, xmm1 |
| punpckhqdq xmm4, xmm1 |
| |
| punpcklqdq xmm2, xmm3 |
| punpckhqdq xmm5, xmm3 |
| |
| movdqa XMMWORD PTR[output + 0 ], xmm0 |
| movdqa XMMWORD PTR[output + 16], xmm2 |
| movdqa XMMWORD PTR[output + 32], xmm4 |
| movdqa XMMWORD PTR[output + 48], xmm5 |
| |
| STACK_FRAME_DESTROY |
| |
| SECTION_RODATA |
| align 16 |
| _5352_2217: |
| dw 5352 |
| dw 2217 |
| dw 5352 |
| dw 2217 |
| dw 5352 |
| dw 2217 |
| dw 5352 |
| dw 2217 |
| align 16 |
| _2217_neg5352: |
| dw 2217 |
| dw -5352 |
| dw 2217 |
| dw -5352 |
| dw 2217 |
| dw -5352 |
| dw 2217 |
| dw -5352 |
| align 16 |
| _mult_add: |
| times 8 dw 1 |
| align 16 |
| _cmp_mask: |
| times 4 dw 1 |
| times 4 dw 0 |
| align 16 |
| _cmp_mask8x4: |
| times 8 dw 1 |
| align 16 |
| _mult_sub: |
| dw 1 |
| dw -1 |
| dw 1 |
| dw -1 |
| dw 1 |
| dw -1 |
| dw 1 |
| dw -1 |
| align 16 |
| _7: |
| times 4 dd 7 |
| align 16 |
| _7w: |
| times 8 dw 7 |
| align 16 |
| _14500: |
| times 4 dd 14500 |
| align 16 |
| _7500: |
| times 4 dd 7500 |
| align 16 |
| _12000: |
| times 4 dd 12000 |
| align 16 |
| _51000: |
| times 4 dd 51000 |
