| ; Copyright © 2021, VideoLAN and dav1d authors |
| ; Copyright © 2021, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "config.asm" |
| %include "ext/x86/x86inc.asm" |
| |
| %if ARCH_X86_64 |
| |
| SECTION_RODATA 32 |
| |
| wiener_shufA: db 1, 2, 7, 6, 3, 4, 9, 8, 5, 6, 11, 10, 7, 8, 13, 12 |
| wiener_shufB: db 2, 3, 8, 7, 4, 5, 10, 9, 6, 7, 12, 11, 8, 9, 14, 13 |
| wiener_shufC: db 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11 |
| wiener_shufD: db 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 |
| wiener_perm32: db 1, 9, 3, 11, 5, 13, 7, 15, 33, 41, 35, 43, 37, 45, 39, 47 |
| db 17, 25, 19, 27, 21, 29, 23, 31, 49, 57, 51, 59, 53, 61, 55, 63 |
| sgr_shuf: db 128, 1, -1, 2,132, 3, -1, 4,136, 5, -1, 6,140, 7, -1, 8 |
| db 129, 9, -1, 10,133, 11, -1, 12,137, -1, -1, -1,141, -1, 0,128 |
| sgr_mix_perm: db 1, 3, 5, 7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55 |
| r_ext_mask: times 68 db -1 |
| times 4 db 0 |
| wiener_x_shuf: db 0, 2, -1, 0 |
| wiener_x_add: db 0, 1,127, 0 |
| |
| pw_61448: times 2 dw 61448 |
| pw_164_455: dw 164, 455 |
| pd_m16380: dd -16380 |
| pd_m4096: dd -4096 |
| pd_m25 dd -25 |
| pd_m9: dd -9 |
| pd_34816: dd 34816 |
| pd_8421376: dd 8421376 |
| |
| cextern sgr_x_by_x |
| |
| SECTION .text |
| |
| DECLARE_REG_TMP 8, 7, 9, 11, 12, 13, 14 ; ring buffer pointers |
| |
| INIT_ZMM avx512icl |
| cglobal wiener_filter7_8bpc, 4, 15, 20, -384*12-16, dst, stride, left, lpf, \ |
| w, h, edge, flt |
| mov fltq, r6mp |
| mov wd, wm |
| movifnidn hd, hm |
| mov edged, r7m |
| vbroadcasti32x4 m6, [wiener_shufA] |
| vbroadcasti32x4 m7, [wiener_shufB] |
| mov r10d, 0xfffe |
| vbroadcasti32x4 m8, [wiener_shufC] |
| vbroadcasti32x4 m9, [wiener_shufD] |
| kmovw k1, r10d |
| vpbroadcastd m0, [wiener_x_shuf] |
| vpbroadcastd m1, [wiener_x_add] |
| mov r10, 0xaaaaaaaaaaaaaaaa |
| vpbroadcastd m11, [fltq+ 0] |
| vpbroadcastd m12, [fltq+ 4] |
| kmovq k2, r10 |
| vpbroadcastd m10, [pd_m16380] |
| packsswb m11, m11 ; x0 x1 x0 x1 |
| vpbroadcastd m14, [fltq+16] |
| pshufb m12, m0 |
| vpbroadcastd m15, [fltq+20] |
| paddb m12, m1 ; x2 x3+1 x2 127 |
| vpbroadcastd m13, [pd_8421376] |
| psllw m14, 5 ; y0 y1 |
| psllw m15, 5 ; y2 y3 |
| cmp wd, 32 ; the minimum lr unit size for chroma in 4:2:0 is 32 |
| jle .w32 ; pixels, so we need a special case for small widths |
| lea t1, [rsp+wq*2+16] |
| add lpfq, wq |
| add dstq, wq |
| neg wq |
| test edgeb, 4 ; LR_HAVE_TOP |
| jz .no_top |
| call .h_top |
| add lpfq, strideq |
| mov t6, t1 |
| mov t5, t1 |
| add t1, 384*2 |
| call .h_top |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| mov t4, t1 |
| add t1, 384*2 |
| add r10, strideq |
| mov [rsp], r10 ; below |
| call .h |
| mov t3, t1 |
| mov t2, t1 |
| dec hd |
| jz .v1 |
| add lpfq, strideq |
| add t1, 384*2 |
| call .h |
| mov t2, t1 |
| dec hd |
| jz .v2 |
| add lpfq, strideq |
| add t1, 384*2 |
| call .h |
| dec hd |
| jz .v3 |
| .main: |
| lea t0, [t1+384*2] |
| .main_loop: |
| call .hv |
| dec hd |
| jnz .main_loop |
| test edgeb, 8 ; LR_HAVE_BOTTOM |
| jz .v3 |
| mov lpfq, [rsp] |
| call .hv_bottom |
| add lpfq, strideq |
| call .hv_bottom |
| .v1: |
| call .v |
| RET |
| .no_top: |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| lea r10, [r10+strideq*2] |
| mov [rsp], r10 |
| call .h |
| mov t6, t1 |
| mov t5, t1 |
| mov t4, t1 |
| mov t3, t1 |
| mov t2, t1 |
| dec hd |
| jz .v1 |
| add lpfq, strideq |
| add t1, 384*2 |
| call .h |
| mov t2, t1 |
| dec hd |
| jz .v2 |
| add lpfq, strideq |
| add t1, 384*2 |
| call .h |
| dec hd |
| jz .v3 |
| lea t0, [t1+384*2] |
| call .hv |
| dec hd |
| jz .v3 |
| add t0, 384*8 |
| call .hv |
| dec hd |
| jnz .main |
| .v3: |
| call .v |
| .v2: |
| call .v |
| jmp .v1 |
| .h: |
| mov r10, wq |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| movd xm16, [leftq] |
| vmovdqu32 m16{k1}, [lpfq+r10-4] |
| add leftq, 4 |
| jmp .h_main |
| .h_extend_left: |
| vpbroadcastb xm16, [lpfq+r10] ; the masked load ensures that no exception |
| vmovdqu32 m16{k1}, [lpfq+r10-4] ; gets raised from accessing invalid memory |
| jmp .h_main |
| .h_top: |
| mov r10, wq |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| .h_loop: |
| movu m16, [lpfq+r10-4] |
| .h_main: |
| movu m17, [lpfq+r10+4] |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .h_have_right |
| cmp r10d, -66 |
| jl .h_have_right |
| push r0 |
| lea r0, [r_ext_mask+65] |
| vpbroadcastb m0, [lpfq-1] |
| vpternlogd m16, m0, [r0+r10+0], 0xe4 ; c ? a : b |
| vpternlogd m17, m0, [r0+r10+8], 0xe4 |
| pop r0 |
| .h_have_right: |
| pshufb m4, m16, m6 |
| mova m0, m10 |
| vpdpbusd m0, m4, m11 |
| pshufb m4, m16, m7 |
| mova m2, m10 |
| vpdpbusd m2, m4, m11 |
| pshufb m4, m17, m6 |
| mova m1, m10 |
| vpdpbusd m1, m4, m11 |
| pshufb m4, m17, m7 |
| mova m3, m10 |
| vpdpbusd m3, m4, m11 |
| pshufb m4, m16, m8 |
| vpdpbusd m0, m4, m12 |
| pshufb m16, m9 |
| vpdpbusd m2, m16, m12 |
| pshufb m4, m17, m8 |
| vpdpbusd m1, m4, m12 |
| pshufb m17, m9 |
| vpdpbusd m3, m17, m12 |
| packssdw m0, m2 |
| packssdw m1, m3 |
| psraw m0, 3 |
| psraw m1, 3 |
| mova [t1+r10*2+ 0], m0 |
| mova [t1+r10*2+64], m1 |
| add r10, 64 |
| jl .h_loop |
| ret |
| ALIGN function_align |
| .hv: |
| add lpfq, strideq |
| mov r10, wq |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| movd xm16, [leftq] |
| vmovdqu32 m16{k1}, [lpfq+r10-4] |
| add leftq, 4 |
| jmp .hv_main |
| .hv_extend_left: |
| vpbroadcastb xm16, [lpfq+r10] |
| vmovdqu32 m16{k1}, [lpfq+r10-4] |
| jmp .hv_main |
| .hv_bottom: |
| mov r10, wq |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| .hv_loop: |
| movu m16, [lpfq+r10-4] |
| .hv_main: |
| movu m17, [lpfq+r10+4] |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .hv_have_right |
| cmp r10d, -66 |
| jl .hv_have_right |
| push r0 |
| lea r0, [r_ext_mask+65] |
| vpbroadcastb m0, [lpfq-1] |
| vpternlogd m16, m0, [r0+r10+0], 0xe4 ; c ? a : b |
| vpternlogd m17, m0, [r0+r10+8], 0xe4 |
| pop r0 |
| .hv_have_right: |
| pshufb m4, m16, m6 |
| mova m0, m10 |
| vpdpbusd m0, m4, m11 |
| pshufb m4, m16, m7 |
| mova m2, m10 |
| vpdpbusd m2, m4, m11 |
| pshufb m4, m17, m6 |
| mova m1, m10 |
| vpdpbusd m1, m4, m11 |
| pshufb m4, m17, m7 |
| mova m3, m10 |
| vpdpbusd m3, m4, m11 |
| pshufb m4, m16, m8 |
| vpdpbusd m0, m4, m12 |
| pshufb m16, m9 |
| vpdpbusd m2, m16, m12 |
| pshufb m4, m17, m8 |
| vpdpbusd m1, m4, m12 |
| pshufb m17, m9 |
| vpdpbusd m3, m17, m12 |
| packssdw m0, m2 |
| packssdw m1, m3 |
| psraw m0, 3 |
| psraw m1, 3 |
| mova m16, [t4+r10*2] |
| paddw m16, [t2+r10*2] |
| mova m3, [t3+r10*2] |
| mova m17, [t4+r10*2+64] |
| paddw m17, [t2+r10*2+64] |
| mova m5, [t3+r10*2+64] |
| punpcklwd m4, m16, m3 |
| mova m2, m13 |
| vpdpwssd m2, m4, m15 |
| punpcklwd m18, m17, m5 |
| mova m4, m13 |
| vpdpwssd m4, m18, m15 |
| punpckhwd m16, m3 |
| mova m3, m13 |
| vpdpwssd m3, m16, m15 |
| punpckhwd m17, m5 |
| mova m5, m13 |
| vpdpwssd m5, m17, m15 |
| mova m17, [t5+r10*2] |
| paddw m17, [t1+r10*2] |
| paddw m16, m0, [t6+r10*2] |
| mova m19, [t5+r10*2+64] |
| paddw m19, [t1+r10*2+64] |
| paddw m18, m1, [t6+r10*2+64] |
| mova [t0+r10*2+ 0], m0 |
| mova [t0+r10*2+64], m1 |
| punpcklwd m0, m16, m17 |
| vpdpwssd m2, m0, m14 |
| punpcklwd m1, m18, m19 |
| vpdpwssd m4, m1, m14 |
| punpckhwd m16, m17 |
| vpdpwssd m3, m16, m14 |
| punpckhwd m18, m19 |
| vpdpwssd m5, m18, m14 |
| packuswb m2, m4 |
| psrlw m2, 8 |
| vpackuswb m2{k2}, m3, m5 |
| movu [dstq+r10], m2 ; We don't have a separate 5-tap version so the 7-tap |
| add r10, 64 ; function is used for chroma as well, and in some |
| jl .hv_loop ; esoteric edge cases chroma dst pointers may only |
| mov t6, t5 ; have a 32-byte alignment despite having a width |
| mov t5, t4 ; larger than 32, so use an unaligned store here. |
| mov t4, t3 |
| mov t3, t2 |
| mov t2, t1 |
| mov t1, t0 |
| mov t0, t6 |
| add dstq, strideq |
| ret |
| .v: |
| mov r10, wq |
| .v_loop: |
| mova m4, [t4+r10*2+ 0] |
| paddw m4, [t2+r10*2+ 0] |
| mova m1, [t3+r10*2+ 0] |
| mova m5, [t4+r10*2+64] |
| paddw m5, [t2+r10*2+64] |
| mova m3, [t3+r10*2+64] |
| punpcklwd m6, m4, m1 |
| mova m0, m13 |
| vpdpwssd m0, m6, m15 |
| punpcklwd m6, m5, m3 |
| mova m2, m13 |
| vpdpwssd m2, m6, m15 |
| punpckhwd m4, m1 |
| mova m1, m13 |
| vpdpwssd m1, m4, m15 |
| punpckhwd m5, m3 |
| mova m3, m13 |
| vpdpwssd m3, m5, m15 |
| mova m5, [t1+r10*2+ 0] |
| paddw m4, m5, [t6+r10*2+ 0] |
| paddw m5, [t5+r10*2+ 0] |
| mova m7, [t1+r10*2+64] |
| paddw m6, m7, [t6+r10*2+64] |
| paddw m7, [t5+r10*2+64] |
| punpcklwd m8, m4, m5 |
| vpdpwssd m0, m8, m14 |
| punpcklwd m8, m6, m7 |
| vpdpwssd m2, m8, m14 |
| punpckhwd m4, m5 |
| vpdpwssd m1, m4, m14 |
| punpckhwd m6, m7 |
| vpdpwssd m3, m6, m14 |
| packuswb m0, m2 |
| psrlw m0, 8 |
| vpackuswb m0{k2}, m1, m3 |
| movu [dstq+r10], m0 |
| add r10, 64 |
| jl .v_loop |
| mov t6, t5 |
| mov t5, t4 |
| mov t4, t3 |
| mov t3, t2 |
| mov t2, t1 |
| add dstq, strideq |
| ret |
| .w32: |
| lea r10, [r_ext_mask+73] |
| mova ym18, [wiener_perm32] |
| lea t1, [rsp+16] |
| sub r10, wq |
| test edgeb, 4 ; LR_HAVE_TOP |
| jz .w32_no_top |
| call .w32_h_top |
| add lpfq, strideq |
| mov t6, t1 |
| mov t5, t1 |
| add t1, 32*2 |
| call .w32_h_top |
| lea r9, [lpfq+strideq*4] |
| mov lpfq, dstq |
| mov t4, t1 |
| add t1, 32*2 |
| add r9, strideq |
| mov [rsp], r9 ; below |
| call .w32_h |
| mov t3, t1 |
| mov t2, t1 |
| dec hd |
| jz .w32_v1 |
| add lpfq, strideq |
| add t1, 32*2 |
| call .w32_h |
| mov t2, t1 |
| dec hd |
| jz .w32_v2 |
| add lpfq, strideq |
| add t1, 32*2 |
| call .w32_h |
| dec hd |
| jz .w32_v3 |
| .w32_main: |
| lea t0, [t1+32*2] |
| .w32_main_loop: |
| call .w32_hv |
| dec hd |
| jnz .w32_main_loop |
| test edgeb, 8 ; LR_HAVE_BOTTOM |
| jz .w32_v3 |
| mov lpfq, [rsp] |
| call .w32_hv_bottom |
| add lpfq, strideq |
| call .w32_hv_bottom |
| .w32_v1: |
| call .w32_v |
| RET |
| .w32_no_top: |
| lea r9, [lpfq+strideq*4] |
| mov lpfq, dstq |
| lea r9, [r9+strideq*2] |
| mov [rsp], r9 |
| call .w32_h |
| mov t6, t1 |
| mov t5, t1 |
| mov t4, t1 |
| mov t3, t1 |
| mov t2, t1 |
| dec hd |
| jz .w32_v1 |
| add lpfq, strideq |
| add t1, 32*2 |
| call .w32_h |
| mov t2, t1 |
| dec hd |
| jz .w32_v2 |
| add lpfq, strideq |
| add t1, 32*2 |
| call .w32_h |
| dec hd |
| jz .w32_v3 |
| lea t0, [t1+32*2] |
| call .w32_hv |
| dec hd |
| jz .w32_v3 |
| add t0, 32*8 |
| call .w32_hv |
| dec hd |
| jnz .w32_main |
| .w32_v3: |
| call .w32_v |
| .w32_v2: |
| call .w32_v |
| jmp .w32_v1 |
| .w32_h: |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .w32_h_extend_left |
| movd xm16, [leftq] |
| vmovdqu32 ym16{k1}, [lpfq-4] |
| add leftq, 4 |
| jmp .w32_h_main |
| .w32_h_extend_left: |
| vpbroadcastb xm16, [lpfq] ; the masked load ensures that no exception |
| vmovdqu32 ym16{k1}, [lpfq-4] ; gets raised from accessing invalid memory |
| jmp .w32_h_main |
| .w32_h_top: |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .w32_h_extend_left |
| movu ym16, [lpfq-4] |
| .w32_h_main: |
| vinserti32x8 m16, [lpfq+4], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .w32_h_have_right |
| vpbroadcastb m0, [lpfq+wq-1] |
| movu ym17, [r10-8] |
| vinserti32x8 m17, [r10+0], 1 |
| vpternlogd m16, m0, m17, 0xe4 ; c ? a : b |
| .w32_h_have_right: |
| pshufb m2, m16, m6 |
| mova m0, m10 |
| vpdpbusd m0, m2, m11 |
| pshufb m2, m16, m7 |
| mova m1, m10 |
| vpdpbusd m1, m2, m11 |
| pshufb m2, m16, m8 |
| vpdpbusd m0, m2, m12 |
| pshufb m16, m9 |
| vpdpbusd m1, m16, m12 |
| packssdw m0, m1 |
| psraw m0, 3 |
| mova [t1], m0 |
| ret |
| .w32_hv: |
| add lpfq, strideq |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .w32_hv_extend_left |
| movd xm16, [leftq] |
| vmovdqu32 ym16{k1}, [lpfq-4] |
| add leftq, 4 |
| jmp .w32_hv_main |
| .w32_hv_extend_left: |
| vpbroadcastb xm16, [lpfq] |
| vmovdqu32 ym16{k1}, [lpfq-4] |
| jmp .w32_hv_main |
| .w32_hv_bottom: |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .w32_hv_extend_left |
| movu ym16, [lpfq-4] |
| .w32_hv_main: |
| vinserti32x8 m16, [lpfq+4], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .w32_hv_have_right |
| vpbroadcastb m0, [lpfq+wq-1] |
| movu ym17, [r10-8] |
| vinserti32x8 m17, [r10+0], 1 |
| vpternlogd m16, m0, m17, 0xe4 |
| .w32_hv_have_right: |
| mova m3, [t4] |
| paddw m3, [t2] |
| mova m2, [t3] |
| pshufb m4, m16, m6 |
| mova m0, m10 |
| vpdpbusd m0, m4, m11 |
| pshufb m4, m16, m7 |
| mova m5, m10 |
| vpdpbusd m5, m4, m11 |
| punpcklwd m4, m3, m2 |
| mova m1, m13 |
| vpdpwssd m1, m4, m15 |
| punpckhwd m3, m2 |
| mova m2, m13 |
| vpdpwssd m2, m3, m15 |
| pshufb m4, m16, m8 |
| vpdpbusd m0, m4, m12 |
| pshufb m16, m9 |
| vpdpbusd m5, m16, m12 |
| packssdw m0, m5 |
| psraw m0, 3 |
| mova m4, [t5] |
| paddw m4, [t1] |
| paddw m3, m0, [t6] |
| mova [t0], m0 |
| punpcklwd m0, m3, m4 |
| vpdpwssd m1, m0, m14 |
| punpckhwd m3, m4 |
| vpdpwssd m2, m3, m14 |
| packuswb m1, m2 |
| vpermb m16, m18, m1 |
| mova [dstq], ym16 |
| mov t6, t5 |
| mov t5, t4 |
| mov t4, t3 |
| mov t3, t2 |
| mov t2, t1 |
| mov t1, t0 |
| mov t0, t6 |
| add dstq, strideq |
| ret |
| .w32_v: |
| mova m2, [t4] |
| paddw m2, [t2] |
| mova m1, [t3] |
| mova m4, [t1] |
| paddw m3, m4, [t6] |
| paddw m4, [t5] |
| punpcklwd m5, m2, m1 |
| mova m0, m13 |
| vpdpwssd m0, m5, m15 |
| punpckhwd m2, m1 |
| mova m1, m13 |
| vpdpwssd m1, m2, m15 |
| punpcklwd m2, m3, m4 |
| vpdpwssd m0, m2, m14 |
| punpckhwd m3, m4 |
| vpdpwssd m1, m3, m14 |
| packuswb m0, m1 |
| vpermb m16, m18, m0 |
| mova [dstq], ym16 |
| mov t6, t5 |
| mov t5, t4 |
| mov t4, t3 |
| mov t3, t2 |
| mov t2, t1 |
| add dstq, strideq |
| ret |
| |
| cglobal sgr_filter_5x5_8bpc, 4, 13, 23, 416*24+16, dst, stride, left, lpf, \ |
| w, h, edge, params |
| mov paramsq, r6mp |
| mov wd, wm |
| mov hd, hm |
| mov edged, r7m |
| vbroadcasti32x4 m5, [sgr_shuf+1] |
| add lpfq, wq |
| vbroadcasti32x4 m6, [sgr_shuf+9] |
| add dstq, wq |
| vbroadcasti32x4 m7, [sgr_shuf+3] |
| lea t3, [rsp+wq*4+16+416*12] |
| vbroadcasti32x4 m8, [sgr_shuf+7] |
| pxor m4, m4 |
| vpbroadcastd m9, [pd_m25] |
| vpsubd m11, m4, [paramsq+0] {1to16} ; -s0 |
| vpbroadcastw m15, [paramsq+8] ; w0 |
| lea t1, [rsp+wq*2+20] |
| vpbroadcastd m10, [pw_164_455] |
| neg wq |
| vpbroadcastd m12, [pw_61448] ; (15 << 12) + (1 << 3) |
| mov r10d, 0xfe |
| vpbroadcastd m13, [pd_m4096] |
| kmovb k1, r10d |
| vpbroadcastd m14, [pd_34816] ; (1 << 11) + (1 << 15) |
| mov r10, 0x3333333333333333 |
| mova m18, [sgr_x_by_x+64*0] |
| kmovq k2, r10 |
| mova m19, [sgr_x_by_x+64*1] |
| lea r12, [r_ext_mask+75] |
| mova m20, [sgr_x_by_x+64*2] |
| psllw m15, 4 |
| mova m21, [sgr_x_by_x+64*3] |
| lea r10, [lpfq+strideq*4] |
| mova ym22, [sgr_shuf] |
| add r10, strideq |
| mov [rsp], r10 ; below |
| test edgeb, 4 ; LR_HAVE_TOP |
| jz .no_top |
| call .h_top |
| add lpfq, strideq |
| mov t2, t1 |
| call .top_fixup |
| add t1, 416*6 |
| call .h_top |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| add r10, strideq |
| mov [rsp], r10 ; below |
| mov t0, t2 |
| dec hd |
| jz .height1 |
| or edged, 16 |
| call .h |
| .main: |
| add lpfq, strideq |
| call .hv |
| call .prep_n |
| sub hd, 2 |
| jl .extend_bottom |
| .main_loop: |
| add lpfq, strideq |
| test hd, hd |
| jz .odd_height |
| call .h |
| add lpfq, strideq |
| call .hv |
| call .n0 |
| call .n1 |
| sub hd, 2 |
| jge .main_loop |
| test edgeb, 8 ; LR_HAVE_BOTTOM |
| jz .extend_bottom |
| mov lpfq, [rsp] |
| call .h_top |
| add lpfq, strideq |
| call .hv_bottom |
| .end: |
| call .n0 |
| call .n1 |
| .end2: |
| RET |
| .height1: |
| call .hv |
| call .prep_n |
| jmp .odd_height_end |
| .odd_height: |
| call .hv |
| call .n0 |
| call .n1 |
| .odd_height_end: |
| call .v |
| call .n0 |
| jmp .end2 |
| .extend_bottom: |
| call .v |
| jmp .end |
| .no_top: |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| lea r10, [r10+strideq*2] |
| mov [rsp], r10 |
| call .h |
| lea t2, [t1+416*6] |
| call .top_fixup |
| dec hd |
| jz .no_top_height1 |
| or edged, 16 |
| mov t0, t1 |
| mov t1, t2 |
| jmp .main |
| .no_top_height1: |
| call .v |
| call .prep_n |
| jmp .odd_height_end |
| .h: ; horizontal boxsum |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .h_main |
| .h_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .h_main |
| .h_top: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| .h_loop: |
| movu ym17, [lpfq+r10-2] |
| .h_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .h_have_right |
| cmp r10d, -34 |
| jl .h_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r12+r10-8] |
| vinserti32x8 m16, [r12+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .h_have_right: |
| pshufb m3, m17, m5 |
| pmullw m2, m3, m3 |
| pshufb m1, m17, m6 |
| paddw m0, m3, m1 |
| shufps m3, m1, q2121 |
| paddw m0, m3 |
| punpcklwd m16, m3, m1 |
| punpckhwd m3, m1 |
| punpcklwd m1, m2, m4 |
| vpdpwssd m1, m16, m16 |
| punpckhwd m2, m4 |
| vpdpwssd m2, m3, m3 |
| pshufb m16, m17, m7 |
| paddw m0, m16 |
| pshufb m17, m8 |
| paddw m0, m17 ; sum |
| punpcklwd m3, m16, m17 |
| vpdpwssd m1, m3, m3 ; sumsq |
| punpckhwd m16, m17 |
| vpdpwssd m2, m16, m16 |
| test edgeb, 16 ; y > 0 |
| jz .h_loop_end |
| paddw m0, [t1+r10*2+416*0] |
| paddd m1, [t1+r10*2+416*2] |
| paddd m2, [t1+r10*2+416*4] |
| .h_loop_end: |
| mova [t1+r10*2+416*0], m0 |
| mova [t1+r10*2+416*2], m1 |
| mova [t1+r10*2+416*4], m2 |
| add r10, 32 |
| jl .h_loop |
| ret |
| .top_fixup: |
| lea r10, [wq-2] |
| .top_fixup_loop: ; the sums of the first row needs to be doubled |
| mova m0, [t1+r10*2+416*0] |
| mova m1, [t1+r10*2+416*2] |
| mova m2, [t1+r10*2+416*4] |
| paddw m0, m0 |
| paddd m1, m1 |
| paddd m2, m2 |
| mova [t2+r10*2+416*0], m0 |
| mova [t2+r10*2+416*2], m1 |
| mova [t2+r10*2+416*4], m2 |
| add r10, 32 |
| jl .top_fixup_loop |
| ret |
| ALIGN function_align |
| .hv: ; horizontal boxsum + vertical boxsum + ab |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .hv_main |
| .hv_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .hv_main |
| .hv_bottom: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| .hv_loop: |
| movu ym17, [lpfq+r10-2] |
| .hv_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .hv_have_right |
| cmp r10d, -34 |
| jl .hv_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r12+r10-8] |
| vinserti32x8 m16, [r12+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .hv_have_right: |
| pshufb m1, m17, m5 |
| pmullw m3, m1, m1 |
| pshufb m2, m17, m6 |
| paddw m0, m1, m2 |
| shufps m1, m2, q2121 |
| paddw m0, m1 |
| punpcklwd m16, m1, m2 |
| punpckhwd m1, m2 |
| punpcklwd m2, m3, m4 |
| vpdpwssd m2, m16, m16 |
| punpckhwd m3, m4 |
| vpdpwssd m3, m1, m1 |
| pshufb m16, m17, m7 |
| paddw m0, m16 |
| pshufb m17, m8 |
| paddw m0, m17 ; h sum |
| punpcklwd m1, m16, m17 |
| vpdpwssd m2, m1, m1 ; h sumsq |
| punpckhwd m16, m17 |
| vpdpwssd m3, m16, m16 |
| paddw m1, m0, [t1+r10*2+416*0] |
| paddd m16, m2, [t1+r10*2+416*2] |
| paddd m17, m3, [t1+r10*2+416*4] |
| test hd, hd |
| jz .hv_last_row |
| .hv_main2: |
| paddd m16, [t2+r10*2+416*2] ; hv sumsq |
| paddd m17, [t2+r10*2+416*4] |
| paddw m1, [t2+r10*2+416*0] ; hv sum |
| mova [t0+r10*2+416*2], m2 |
| mova [t0+r10*2+416*4], m3 |
| mova [t0+r10*2+416*0], m0 |
| pmulld m16, m9 ; -a * 25 |
| pmulld m17, m9 |
| punpcklwd m0, m1, m4 ; b |
| vpdpwssd m16, m0, m0 ; -p |
| punpckhwd m1, m4 |
| vpdpwssd m17, m1, m1 |
| pmaddwd m0, m10 ; b * 164 |
| pmaddwd m1, m10 |
| pmulld m16, m11 ; p * s |
| pmulld m17, m11 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m20 |
| paddusw m17, m12 |
| psraw m17, 4 ; min(z, 255) - 256 |
| vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x |
| pandn m16, m13, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) |
| paddd m1, m14 |
| vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) |
| vpternlogd m17, m1, m13, 0xd8 |
| mova [t3+r10*4+ 8], m16 ; The neighbor calculations requires |
| mova [t3+r10*4+ 24], xm17 ; 13 bits for a and 21 bits for b. |
| vextracti32x4 [t3+r10*4+ 56], m17, 2 ; Packing them allows for 12+20, but |
| mova [t3+r10*4+ 72], m17 ; that gets us most of the way. |
| vextracti128 [t3+r10*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+104], m16, 3 |
| add r10, 32 |
| jl .hv_loop |
| mov t2, t1 |
| mov t1, t0 |
| mov t0, t2 |
| ret |
| .hv_last_row: ; esoteric edge case for odd heights |
| mova [t1+r10*2+416*0], m1 |
| paddw m1, m0 |
| mova [t1+r10*2+416*2], m16 |
| paddd m16, m2 |
| mova [t1+r10*2+416*4], m17 |
| paddd m17, m3 |
| jmp .hv_main2 |
| .v: ; vertical boxsum + ab |
| lea r10, [wq-2] |
| .v_loop: |
| mova m2, [t1+r10*2+416*2] |
| paddd m16, m2, [t2+r10*2+416*2] |
| mova m3, [t1+r10*2+416*4] |
| paddd m17, m3, [t2+r10*2+416*4] |
| paddd m2, m2 |
| paddd m3, m3 |
| paddd m16, m2 ; hv sumsq |
| paddd m17, m3 |
| pmulld m16, m9 ; -a * 25 |
| pmulld m17, m9 |
| mova m0, [t1+r10*2+416*0] |
| paddw m1, m0, [t2+r10*2+416*0] |
| paddw m0, m0 |
| paddw m1, m0 ; hv sum |
| punpcklwd m0, m1, m4 ; b |
| vpdpwssd m16, m0, m0 ; -p |
| punpckhwd m1, m4 |
| vpdpwssd m17, m1, m1 |
| pmaddwd m0, m10 ; b * 164 |
| pmaddwd m1, m10 |
| pmulld m16, m11 ; p * s |
| pmulld m17, m11 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m20 |
| paddusw m17, m12 |
| psraw m17, 4 ; min(z, 255) - 256 |
| vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x |
| pandn m16, m13, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) |
| paddd m1, m14 |
| vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) |
| vpternlogd m17, m1, m13, 0xd8 |
| mova [t3+r10*4+ 8], m16 |
| mova [t3+r10*4+ 24], xm17 |
| vextracti32x4 [t3+r10*4+ 56], m17, 2 |
| mova [t3+r10*4+ 72], m17 |
| vextracti128 [t3+r10*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+104], m16, 3 |
| add r10, 32 |
| jl .v_loop |
| ret |
| .prep_n: ; initial neighbor setup |
| mov r10, wq |
| .prep_n_loop: |
| movu m0, [t3+r10*4+ 4] |
| movu m1, [t3+r10*4+68] |
| paddd m2, m0, [t3+r10*4+ 0] |
| paddd m3, m1, [t3+r10*4+64] |
| paddd m2, [t3+r10*4+ 8] |
| paddd m3, [t3+r10*4+72] |
| paddd m0, m2 |
| pslld m2, 2 |
| paddd m1, m3 |
| pslld m3, 2 |
| paddd m2, m0 ; ab 565 |
| paddd m3, m1 |
| pandn m0, m13, m2 ; a |
| psrld m2, 12 ; b |
| pandn m1, m13, m3 |
| psrld m3, 12 |
| mova [t3+r10*4+416*4+ 0], m0 |
| mova [t3+r10*4+416*8+ 0], m2 |
| mova [t3+r10*4+416*4+64], m1 |
| mova [t3+r10*4+416*8+64], m3 |
| add r10, 32 |
| jl .prep_n_loop |
| ret |
| ALIGN function_align |
| .n0: ; neighbor + output (even rows) |
| mov r10, wq |
| .n0_loop: |
| movu m16, [t3+r10*4+ 4] |
| movu m17, [t3+r10*4+68] |
| paddd m0, m16, [t3+r10*4+ 0] |
| paddd m1, m17, [t3+r10*4+64] |
| paddd m0, [t3+r10*4+ 8] |
| paddd m1, [t3+r10*4+72] |
| paddd m16, m0 |
| pslld m0, 2 |
| paddd m17, m1 |
| pslld m1, 2 |
| paddd m0, m16 |
| paddd m1, m17 |
| pandn m16, m13, m0 |
| psrld m0, 12 |
| pandn m17, m13, m1 |
| psrld m1, 12 |
| paddd m2, m16, [t3+r10*4+416*4+ 0] ; a |
| paddd m3, m17, [t3+r10*4+416*4+64] |
| mova [t3+r10*4+416*4+ 0], m16 |
| mova [t3+r10*4+416*4+64], m17 |
| paddd m16, m0, [t3+r10*4+416*8+ 0] ; b + (1 << 8) |
| paddd m17, m1, [t3+r10*4+416*8+64] |
| mova [t3+r10*4+416*8+ 0], m0 |
| mova [t3+r10*4+416*8+64], m1 |
| pmovzxbd m0, [dstq+r10+ 0] |
| pmovzxbd m1, [dstq+r10+16] |
| pmaddwd m2, m0 ; a * src |
| pmaddwd m3, m1 |
| packssdw m0, m1 |
| psubd m16, m2 ; b - a * src + (1 << 8) |
| psubd m17, m3 |
| psrad m16, 9 |
| psrad m17, 9 |
| packssdw m16, m17 |
| pmulhrsw m16, m15 |
| paddw m16, m0 |
| packuswb m16, m16 |
| vpermd m16, m22, m16 |
| mova [dstq+r10], ym16 |
| add r10, 32 |
| jl .n0_loop |
| add dstq, strideq |
| ret |
| ALIGN function_align |
| .n1: ; neighbor + output (odd rows) |
| mov r10, wq |
| .n1_loop: |
| pmovzxbd m0, [dstq+r10+ 0] |
| pmovzxbd m1, [dstq+r10+16] |
| pmaddwd m2, m0, [t3+r10*4+416*4+ 0] ; a * src |
| pmaddwd m3, m1, [t3+r10*4+416*4+64] |
| mova m16, [t3+r10*4+416*8+ 0] ; b + (1 << 7) |
| mova m17, [t3+r10*4+416*8+64] |
| packssdw m0, m1 |
| psubd m16, m2 ; b - a * src + (1 << 7) |
| psubd m17, m3 |
| psrad m16, 8 |
| psrad m17, 8 |
| packssdw m16, m17 |
| pmulhrsw m16, m15 |
| paddw m16, m0 |
| packuswb m16, m16 |
| vpermd m16, m22, m16 |
| mova [dstq+r10], ym16 |
| add r10, 32 |
| jl .n1_loop |
| add dstq, strideq |
| ret |
| |
| cglobal sgr_filter_3x3_8bpc, 4, 15, 22, -416*28-16, dst, stride, left, lpf, \ |
| w, h, edge, params |
| mov paramsq, r6mp |
| mov wd, wm |
| movifnidn hd, hm |
| mov edged, r7m |
| vbroadcasti32x4 m5, [sgr_shuf+3] |
| add lpfq, wq |
| vbroadcasti32x4 m6, [sgr_shuf+5] |
| add dstq, wq |
| vbroadcasti32x4 m7, [sgr_shuf+7] |
| pxor m4, m4 |
| vpbroadcastd m8, [pd_m9] |
| vpsubd m11, m4, [paramsq+4] {1to16} ; -s1 |
| vpbroadcastw m15, [paramsq+10] ; w1 |
| lea t1, [rsp+wq*2+20] |
| vpbroadcastd m10, [pw_164_455] |
| lea t3, [rsp+wq*4+16+416*12] |
| vpbroadcastd m12, [pw_61448] ; (15 << 12) + (1 << 3) |
| neg wq |
| vpbroadcastd m13, [pd_m4096] |
| mov r10d, 0xfe |
| vpbroadcastd m14, [pd_34816] ; (1 << 11) + (1 << 15) |
| kmovb k1, r10d |
| mova m18, [sgr_x_by_x+64*0] |
| mov r10, 0x3333333333333333 |
| mova m19, [sgr_x_by_x+64*1] |
| kmovq k2, r10 |
| mova m20, [sgr_x_by_x+64*2] |
| psllw m15, 4 |
| mova m21, [sgr_x_by_x+64*3] |
| lea r14, [r_ext_mask+75] |
| mova ym9, [sgr_shuf] |
| test edgeb, 4 ; LR_HAVE_TOP |
| jz .no_top |
| call .h_top |
| add lpfq, strideq |
| mov t2, t1 |
| add t1, 416*6 |
| call .h_top |
| lea t4, [lpfq+strideq*4] |
| mov lpfq, dstq |
| add t4, strideq |
| mov [rsp], t4 ; below |
| mov t0, t2 |
| call .hv |
| .main: |
| mov t5, t3 |
| add t3, 416*4 |
| dec hd |
| jz .height1 |
| add lpfq, strideq |
| call .hv |
| call .prep_n |
| dec hd |
| jz .extend_bottom |
| .main_loop: |
| add lpfq, strideq |
| call .hv |
| call .n |
| dec hd |
| jnz .main_loop |
| test edgeb, 8 ; LR_HAVE_BOTTOM |
| jz .extend_bottom |
| mov lpfq, [rsp] |
| call .hv_bottom |
| call .n |
| add lpfq, strideq |
| call .hv_bottom |
| .end: |
| call .n |
| RET |
| .height1: |
| call .v |
| call .prep_n |
| mov t2, t1 |
| call .v |
| jmp .end |
| .extend_bottom: |
| call .v |
| call .n |
| mov t2, t1 |
| call .v |
| jmp .end |
| .no_top: |
| lea t4, [lpfq+strideq*4] |
| mov lpfq, dstq |
| lea t4, [t4+strideq*2] |
| mov [rsp], t4 |
| call .h |
| lea t0, [t1+416*6] |
| mov t2, t1 |
| call .v |
| jmp .main |
| .h: ; horizontal boxsum |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .h_main |
| .h_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .h_main |
| .h_top: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| .h_loop: |
| movu ym17, [lpfq+r10-2] |
| .h_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .h_have_right |
| cmp r10d, -33 |
| jl .h_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r14+r10-8] |
| vinserti32x8 m16, [r14+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .h_have_right: |
| pshufb m0, m17, m5 |
| pmullw m2, m0, m0 |
| pshufb m16, m17, m6 |
| paddw m0, m16 |
| pshufb m17, m7 |
| paddw m0, m17 ; sum |
| punpcklwd m3, m16, m17 |
| punpcklwd m1, m2, m4 |
| vpdpwssd m1, m3, m3 ; sumsq |
| punpckhwd m16, m17 |
| punpckhwd m2, m4 |
| vpdpwssd m2, m16, m16 |
| mova [t1+r10*2+416*0], m0 |
| mova [t1+r10*2+416*2], m1 |
| mova [t1+r10*2+416*4], m2 |
| add r10, 32 |
| jl .h_loop |
| ret |
| ALIGN function_align |
| .hv: ; horizontal boxsum + vertical boxsum + ab |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .hv_main |
| .hv_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .hv_main |
| .hv_bottom: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv_extend_left |
| .hv_loop: |
| movu ym17, [lpfq+r10-2] |
| .hv_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .hv_have_right |
| cmp r10d, -33 |
| jl .hv_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r14+r10-8] |
| vinserti32x8 m16, [r14+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .hv_have_right: |
| pshufb m0, m17, m5 |
| pmullw m3, m0, m0 |
| pshufb m1, m17, m6 |
| paddw m0, m1 |
| pshufb m17, m7 |
| paddw m0, m17 ; h sum |
| punpcklwd m16, m17, m1 |
| punpcklwd m2, m3, m4 |
| vpdpwssd m2, m16, m16 ; h sumsq |
| punpckhwd m17, m1 |
| punpckhwd m3, m4 |
| vpdpwssd m3, m17, m17 |
| paddw m1, m0, [t2+r10*2+416*0] |
| paddw m1, [t1+r10*2+416*0] ; hv sum |
| paddd m16, m2, [t2+r10*2+416*2] |
| paddd m17, m3, [t2+r10*2+416*4] |
| paddd m16, [t1+r10*2+416*2] ; hv sumsq |
| paddd m17, [t1+r10*2+416*4] |
| mova [t0+r10*2+416*0], m0 |
| mova [t0+r10*2+416*2], m2 |
| mova [t0+r10*2+416*4], m3 |
| pmulld m16, m8 ; -a * 9 |
| pmulld m17, m8 |
| punpcklwd m0, m4, m1 ; b |
| vpdpwssd m16, m0, m0 ; -p |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| pmaddwd m0, m10 ; b * 455 |
| pmaddwd m1, m10 |
| pmulld m16, m11 ; p * s |
| pmulld m17, m11 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m20 |
| paddusw m17, m12 |
| psraw m17, 4 ; min(z, 255) - 256 |
| vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x |
| pandn m16, m13, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddd m0, m14 ; x * b * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m14 |
| vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) |
| vpternlogd m17, m1, m13, 0xd8 |
| mova [t3+r10*4+ 8], m16 |
| mova [t3+r10*4+ 24], xm17 |
| vextracti32x4 [t3+r10*4+ 56], m17, 2 |
| mova [t3+r10*4+ 72], m17 |
| vextracti128 [t3+r10*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+104], m16, 3 |
| add r10, 32 |
| jl .hv_loop |
| mov t2, t1 |
| mov t1, t0 |
| mov t0, t2 |
| ret |
| .v: ; vertical boxsum + ab |
| lea r10, [wq-2] |
| .v_loop: |
| mova m16, [t1+r10*2+416*2] |
| mova m17, [t1+r10*2+416*4] |
| paddd m16, m16 |
| paddd m17, m17 |
| paddd m16, [t2+r10*2+416*2] ; hv sumsq |
| paddd m17, [t2+r10*2+416*4] |
| pmulld m16, m8 ; -a * 9 |
| pmulld m17, m8 |
| mova m1, [t1+r10*2+416*0] |
| paddw m1, m1 |
| paddw m1, [t2+r10*2+416*0] ; hv sum |
| punpcklwd m0, m4, m1 ; b |
| vpdpwssd m16, m0, m0 ; -p |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| pmaddwd m0, m10 ; b * 455 |
| pmaddwd m1, m10 |
| pmulld m16, m11 ; p * s |
| pmulld m17, m11 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m20 |
| paddusw m17, m12 |
| psraw m17, 4 ; min(z, 255) - 256 |
| vpermt2b m16, m17, m21 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m18, m19 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x |
| pandn m16, m13, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddd m0, m14 ; x * b * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m14 |
| vpternlogd m16, m0, m13, 0xd8 ; a | (b << 12) |
| vpternlogd m17, m1, m13, 0xd8 |
| mova [t3+r10*4+ 8], m16 |
| mova [t3+r10*4+ 24], xm17 |
| vextracti32x4 [t3+r10*4+ 56], m17, 2 |
| mova [t3+r10*4+ 72], m17 |
| vextracti128 [t3+r10*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+104], m16, 3 |
| add r10, 32 |
| jl .v_loop |
| ret |
| .prep_n: ; initial neighbor setup |
| mov r10, wq |
| mov t4, t3 |
| add t3, 416*4 |
| .prep_n_loop: |
| mova m2, [t5+r10*4+0] |
| mova m3, [t4+r10*4+0] |
| paddd m2, [t5+r10*4+8] |
| paddd m3, [t4+r10*4+8] |
| paddd m0, m2, [t5+r10*4+4] |
| paddd m1, m3, [t4+r10*4+4] |
| pslld m0, 2 |
| paddd m1, m1 ; ab[ 0] 222 |
| psubd m0, m2 ; ab[-1] 343 |
| mova [t3+r10*4+416*4], m1 |
| paddd m1, m1 |
| mova [t5+r10*4], m0 |
| psubd m1, m3 ; ab[ 0] 343 |
| mova [t4+r10*4], m1 |
| add r10, 16 |
| jl .prep_n_loop |
| ret |
| ; a+b are packed together in a single dword, but we can't do the |
| ; full neighbor calculations before splitting them since we don't |
| ; have sufficient precision. The solution is to do the calculations |
| ; in two equal halves and split a and b before doing the final sum. |
| ALIGN function_align |
| .n: ; neighbor + output |
| mov r10, wq |
| .n_loop: |
| mova m16, [t3+r10*4+ 0] |
| paddd m16, [t3+r10*4+ 8] |
| paddd m17, m16, [t3+r10*4+ 4] |
| paddd m17, m17 ; ab[+1] 222 |
| mova m2, [t3+r10*4+416*4+ 0] |
| paddd m0, m2, [t5+r10*4+ 0] ; ab[ 0] 222 + ab[-1] 343 |
| mova m3, [t3+r10*4+416*4+64] |
| paddd m1, m3, [t5+r10*4+64] |
| mova [t3+r10*4+416*4+ 0], m17 |
| paddd m17, m17 |
| psubd m17, m16 ; ab[+1] 343 |
| mova [t5+r10*4+ 0], m17 |
| paddd m2, m17 ; ab[ 0] 222 + ab[+1] 343 |
| mova m16, [t3+r10*4+64] |
| paddd m16, [t3+r10*4+72] |
| paddd m17, m16, [t3+r10*4+68] |
| paddd m17, m17 |
| mova [t3+r10*4+416*4+64], m17 |
| paddd m17, m17 |
| psubd m17, m16 |
| mova [t5+r10*4+64], m17 |
| pandn m16, m13, m0 |
| psrld m0, 12 |
| paddd m3, m17 |
| pandn m17, m13, m2 |
| psrld m2, 12 |
| paddd m16, m17 ; a |
| pandn m17, m13, m1 |
| psrld m1, 12 |
| paddd m0, m2 ; b + (1 << 8) |
| pandn m2, m13, m3 |
| psrld m3, 12 |
| paddd m17, m2 |
| pmovzxbd m2, [dstq+r10+ 0] |
| paddd m1, m3 |
| pmovzxbd m3, [dstq+r10+16] |
| pmaddwd m16, m2 ; a * src |
| pmaddwd m17, m3 |
| packssdw m2, m3 |
| psubd m0, m16 ; b - a * src + (1 << 8) |
| psubd m1, m17 |
| psrad m0, 9 |
| psrad m1, 9 |
| packssdw m0, m1 |
| pmulhrsw m0, m15 |
| paddw m0, m2 |
| packuswb m0, m0 |
| vpermd m16, m9, m0 |
| mova [dstq+r10], ym16 |
| add r10, 32 |
| jl .n_loop |
| mov r10, t5 |
| mov t5, t4 |
| mov t4, r10 |
| add dstq, strideq |
| ret |
| |
| cglobal sgr_filter_mix_8bpc, 4, 13, 28, 416*56+8, dst, stride, left, lpf, \ |
| w, h, edge, params |
| mov paramsq, r6mp |
| mov wd, wm |
| movifnidn hd, hm |
| mov edged, r7m |
| vbroadcasti128 m5, [sgr_shuf+1] |
| add lpfq, wq |
| vbroadcasti128 m6, [sgr_shuf+9] |
| add dstq, wq |
| vbroadcasti128 m7, [sgr_shuf+3] |
| lea t3, [rsp+wq*4+416*24+8] |
| vbroadcasti128 m8, [sgr_shuf+7] |
| pxor m4, m4 |
| vpbroadcastd m9, [pd_m9] |
| vpsubd m11, m4, [paramsq+0] {1to16} ; -s0 |
| vpbroadcastd m14, [pw_61448] |
| vpsubd m12, m4, [paramsq+4] {1to16} ; -s1 |
| vpbroadcastd m26, [paramsq+8] ; w0 w1 |
| lea t1, [rsp+wq*2+12] |
| vpbroadcastd m10, [pd_m25] |
| neg wq |
| vpbroadcastd m13, [pw_164_455] |
| mov r10d, 0xfe |
| vpbroadcastd m15, [pd_34816] |
| kmovb k1, r10d |
| mova m20, [sgr_x_by_x+64*0] |
| mov r10, 0x3333333333333333 |
| mova m21, [sgr_x_by_x+64*1] |
| kmovq k2, r10 |
| mova m22, [sgr_x_by_x+64*2] |
| lea r12, [r_ext_mask+75] |
| mova m23, [sgr_x_by_x+64*3] |
| vpbroadcastd m24, [pd_m4096] |
| vpbroadcastd m25, [sgr_shuf+28] ; 0x8000____ |
| psllw m26, 5 |
| mova xm27, [sgr_mix_perm] |
| test edgeb, 4 ; LR_HAVE_TOP |
| jz .no_top |
| call .h_top |
| add lpfq, strideq |
| mov t2, t1 |
| call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_avx512icl).top_fixup |
| add t1, 416*12 |
| call .h_top |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| add r10, strideq |
| mov [rsp], r10 ; below |
| call .hv0 |
| .main: |
| dec hd |
| jz .height1 |
| add lpfq, strideq |
| call .hv1 |
| call .prep_n |
| sub hd, 2 |
| jl .extend_bottom |
| .main_loop: |
| add lpfq, strideq |
| call .hv0 |
| test hd, hd |
| jz .odd_height |
| add lpfq, strideq |
| call .hv1 |
| call .n0 |
| call .n1 |
| sub hd, 2 |
| jge .main_loop |
| test edgeb, 8 ; LR_HAVE_BOTTOM |
| jz .extend_bottom |
| mov lpfq, [rsp] |
| call .hv0_bottom |
| add lpfq, strideq |
| call .hv1_bottom |
| .end: |
| call .n0 |
| call .n1 |
| .end2: |
| RET |
| .height1: |
| call .v1 |
| call .prep_n |
| jmp .odd_height_end |
| .odd_height: |
| call .v1 |
| call .n0 |
| call .n1 |
| .odd_height_end: |
| call .v0 |
| call .v1 |
| call .n0 |
| jmp .end2 |
| .extend_bottom: |
| call .v0 |
| call .v1 |
| jmp .end |
| .no_top: |
| lea r10, [lpfq+strideq*4] |
| mov lpfq, dstq |
| lea r10, [r10+strideq*2] |
| mov [rsp], r10 |
| call .h |
| lea t2, [t1+416*12] |
| lea r10, [wq-2] |
| .top_fixup_loop: |
| mova m0, [t1+r10*2+416* 0] |
| mova m1, [t1+r10*2+416* 2] |
| mova m2, [t1+r10*2+416* 4] |
| paddw m0, m0 |
| mova m3, [t1+r10*2+416* 6] |
| paddd m1, m1 |
| mova m16, [t1+r10*2+416* 8] |
| paddd m2, m2 |
| mova m17, [t1+r10*2+416*10] |
| mova [t2+r10*2+416* 0], m0 |
| mova [t2+r10*2+416* 2], m1 |
| mova [t2+r10*2+416* 4], m2 |
| mova [t2+r10*2+416* 6], m3 |
| mova [t2+r10*2+416* 8], m16 |
| mova [t2+r10*2+416*10], m17 |
| add r10, 32 |
| jl .top_fixup_loop |
| call .v0 |
| jmp .main |
| .h: ; horizontal boxsums |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .h_main |
| .h_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .h_main |
| .h_top: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .h_extend_left |
| .h_loop: |
| movu ym17, [lpfq+r10-2] |
| .h_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .h_have_right |
| cmp r10d, -34 |
| jl .h_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r12+r10-8] |
| vinserti32x8 m16, [r12+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .h_have_right: |
| pshufb m3, m17, m5 |
| pshufb m18, m17, m6 |
| shufps m0, m3, m18, q2121 |
| pmullw m2, m0, m0 |
| pshufb m19, m17, m7 |
| paddw m0, m19 |
| pshufb m17, m8 |
| paddw m0, m17 ; sum3 |
| punpcklwd m16, m19, m17 |
| punpcklwd m1, m2, m4 |
| vpdpwssd m1, m16, m16 ; sumsq3 |
| punpckhwd m19, m17 |
| punpckhwd m2, m4 |
| vpdpwssd m2, m19, m19 |
| mova [t1+r10*2+416* 6], m0 |
| mova [t1+r10*2+416* 8], m1 |
| mova [t1+r10*2+416*10], m2 |
| punpcklwd m19, m3, m18 |
| paddw m0, m3 |
| vpdpwssd m1, m19, m19 ; sumsq5 |
| punpckhwd m3, m18 |
| paddw m0, m18 ; sum5 |
| vpdpwssd m2, m3, m3 |
| mova [t1+r10*2+416* 0], m0 |
| mova [t1+r10*2+416* 2], m1 |
| mova [t1+r10*2+416* 4], m2 |
| add r10, 32 |
| jl .h_loop |
| ret |
| ALIGN function_align |
| .hv0: ; horizontal boxsums + vertical boxsum3 + ab3 (even rows) |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv0_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .hv0_main |
| .hv0_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .hv0_main |
| .hv0_bottom: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv0_extend_left |
| .hv0_loop: |
| movu ym17, [lpfq+r10-2] |
| .hv0_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .hv0_have_right |
| cmp r10d, -34 |
| jl .hv0_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r12+r10-8] |
| vinserti32x8 m16, [r12+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .hv0_have_right: |
| pshufb m18, m17, m5 |
| pshufb m19, m17, m6 |
| shufps m1, m18, m19, q2121 |
| pmullw m3, m1, m1 |
| pshufb m0, m17, m7 |
| paddw m1, m0 |
| pshufb m17, m8 |
| paddw m1, m17 ; sum3 |
| punpcklwd m16, m0, m17 |
| punpcklwd m2, m3, m4 |
| vpdpwssd m2, m16, m16 ; sumsq3 |
| punpckhwd m0, m17 |
| punpckhwd m3, m4 |
| vpdpwssd m3, m0, m0 |
| paddw m0, m1, [t1+r10*2+416* 6] |
| paddd m16, m2, [t1+r10*2+416* 8] |
| paddd m17, m3, [t1+r10*2+416*10] |
| mova [t1+r10*2+416* 6], m1 |
| mova [t1+r10*2+416* 8], m2 |
| mova [t1+r10*2+416*10], m3 |
| paddw m1, m18 |
| paddw m1, m19 ; sum5 |
| mova [t3+r10*4+416*8+ 8], m1 |
| paddw m1, [t1+r10*2+416* 0] |
| mova [t1+r10*2+416* 0], m1 |
| punpcklwd m1, m18, m19 |
| vpdpwssd m2, m1, m1 ; sumsq5 |
| punpckhwd m18, m19 |
| vpdpwssd m3, m18, m18 |
| mova [t3+r10*4+416*0+ 8], m2 ; we need a clean copy of the last row |
| mova [t3+r10*4+416*0+72], m3 ; in case height is odd |
| paddd m2, [t1+r10*2+416* 2] |
| paddd m3, [t1+r10*2+416* 4] |
| mova [t1+r10*2+416* 2], m2 |
| mova [t1+r10*2+416* 4], m3 |
| paddw m1, m0, [t2+r10*2+416* 6] |
| paddd m2, m16, [t2+r10*2+416* 8] |
| paddd m3, m17, [t2+r10*2+416*10] |
| mova [t2+r10*2+416* 6], m0 |
| mova [t2+r10*2+416* 8], m16 |
| mova [t2+r10*2+416*10], m17 |
| pmulld m16, m2, m9 ; -a3 * 9 |
| pmulld m17, m3, m9 |
| punpcklwd m0, m4, m1 ; b3 |
| vpdpwssd m16, m0, m0 ; -p3 |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| pmulld m16, m12 ; p3 * s1 |
| pmulld m17, m12 |
| pmaddwd m0, m13 ; b3 * 455 |
| pmaddwd m1, m13 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m22 |
| paddusw m17, m14 |
| psraw m17, 4 ; min(z3, 255) - 256 |
| vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x3 |
| pandn m16, m24, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m15 |
| vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) |
| vpternlogd m17, m1, m24, 0xd8 |
| mova [t3+r10*4+416*4+ 8], m16 |
| mova [t3+r10*4+416*4+ 24], xm17 |
| vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2 |
| mova [t3+r10*4+416*4+ 72], m17 |
| vextracti128 [t3+r10*4+416*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+416*4+104], m16, 3 |
| add r10, 32 |
| jl .hv0_loop |
| ret |
| ALIGN function_align |
| .hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv1_extend_left |
| movd xm17, [leftq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| add leftq, 4 |
| jmp .hv1_main |
| .hv1_extend_left: |
| vpbroadcastb xm17, [lpfq+wq] |
| vmovdqu32 ym17{k1}, [lpfq+wq-4] |
| jmp .hv1_main |
| .hv1_bottom: |
| lea r10, [wq-2] |
| test edgeb, 1 ; LR_HAVE_LEFT |
| jz .hv1_extend_left |
| .hv1_loop: |
| movu ym17, [lpfq+r10-2] |
| .hv1_main: |
| vinserti32x8 m17, [lpfq+r10+6], 1 |
| test edgeb, 2 ; LR_HAVE_RIGHT |
| jnz .hv1_have_right |
| cmp r10d, -34 |
| jl .hv1_have_right |
| vpbroadcastb m0, [lpfq-1] |
| movu ym16, [r12+r10-8] |
| vinserti32x8 m16, [r12+r10+0], 1 |
| vpternlogd m17, m0, m16, 0xe4 |
| .hv1_have_right: |
| pshufb m3, m17, m5 |
| pshufb m19, m17, m6 |
| shufps m2, m3, m19, q2121 |
| pmullw m1, m2, m2 |
| pshufb m18, m17, m7 |
| paddw m2, m18 |
| pshufb m17, m8 |
| paddw m2, m17 ; sum3 |
| punpcklwd m16, m17, m18 |
| punpcklwd m0, m1, m4 |
| vpdpwssd m0, m16, m16 ; sumsq3 |
| punpckhwd m17, m18 |
| punpckhwd m1, m4 |
| vpdpwssd m1, m17, m17 |
| paddd m16, m0, [t2+r10*2+416* 8] |
| paddd m17, m1, [t2+r10*2+416*10] |
| mova [t2+r10*2+416* 8], m0 |
| mova [t2+r10*2+416*10], m1 |
| punpcklwd m18, m3, m19 |
| vpdpwssd m0, m18, m18 ; sumsq5 |
| punpckhwd m18, m3, m19 |
| vpdpwssd m1, m18, m18 |
| paddw m3, m19 |
| pmulld m16, m9 ; -a3 * 9 |
| pmulld m17, m9 |
| paddd m18, m0, [t2+r10*2+416*2] |
| paddd m19, m1, [t2+r10*2+416*4] |
| paddd m18, [t1+r10*2+416*2] |
| paddd m19, [t1+r10*2+416*4] |
| mova [t2+r10*2+416*2], m0 |
| mova [t2+r10*2+416*4], m1 |
| pmulld m18, m10 ; -a5 * 25 |
| pmulld m19, m10 |
| paddw m1, m2, [t2+r10*2+416* 6] |
| mova [t2+r10*2+416* 6], m2 |
| paddw m2, m3 ; sum5 |
| paddw m3, m2, [t2+r10*2+416*0] |
| paddw m3, [t1+r10*2+416*0] |
| mova [t2+r10*2+416*0], m2 |
| punpcklwd m0, m4, m1 ; b3 |
| vpdpwssd m16, m0, m0 ; -p3 |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| punpcklwd m2, m3, m4 ; b5 |
| vpdpwssd m18, m2, m2 ; -p5 |
| punpckhwd m3, m4 |
| vpdpwssd m19, m3, m3 |
| pmulld m16, m12 ; p3 * s1 |
| pmulld m17, m12 |
| pmulld m18, m11 ; p5 * s0 |
| pmulld m19, m11 |
| pmaddwd m0, m13 ; b3 * 455 |
| pmaddwd m1, m13 |
| pmaddwd m2, m13 ; b5 * 164 |
| pmaddwd m3, m13 |
| vpalignr m17{k2}, m16, m16, 2 |
| vpalignr m19{k2}, m18, m18, 2 |
| paddusw m17, m14 |
| mova m16, m22 |
| psraw m17, 4 ; min(z3, 255) - 256 |
| vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] |
| paddusw m19, m14 |
| mova m18, m22 |
| psraw m19, 4 ; min(z5, 255) - 256 |
| vpermt2b m18, m19, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k4, m19 |
| vpermi2b m19, m20, m21 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x3 |
| vmovdqu8 m19{k4}, m18 ; x5 |
| pandn m16, m24, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| pandn m18, m24, m19 |
| psrld m19, 16 |
| pmulld m2, m18 |
| pmulld m3, m19 |
| paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m15 |
| vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) |
| vpternlogd m17, m1, m24, 0xd8 |
| mova [t3+r10*4+416*8+ 8], m16 |
| mova [t3+r10*4+416*8+ 24], xm17 |
| vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2 |
| paddd m2, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) |
| paddd m3, m15 |
| mova [t3+r10*4+416*8+ 72], m17 |
| vextracti128 [t3+r10*4+416*8+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+416*8+104], m16, 3 |
| vpternlogd m18, m2, m24, 0xd8 ; a5 | (b5 << 12) |
| vpternlogd m19, m3, m24, 0xd8 |
| mova [t3+r10*4+416*0+ 8], m18 |
| mova [t3+r10*4+416*0+ 24], xm19 |
| vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2 |
| mova [t3+r10*4+416*0+ 72], m19 |
| vextracti128 [t3+r10*4+416*0+ 72], ym18, 1 |
| vextracti32x4 [t3+r10*4+416*0+104], m18, 3 |
| add r10, 32 |
| jl .hv1_loop |
| mov r10, t2 |
| mov t2, t1 |
| mov t1, r10 |
| ret |
| .v0: ; vertical boxsums + ab3 (even rows) |
| lea r10, [wq-2] |
| .v0_loop: |
| mova m2, [t1+r10*2+416* 8] |
| mova m3, [t1+r10*2+416*10] |
| paddd m2, m2 |
| paddd m3, m3 |
| paddd m16, m2, [t2+r10*2+416* 8] |
| paddd m17, m3, [t2+r10*2+416*10] |
| mova m0, [t1+r10*2+416* 6] |
| paddw m0, m0 |
| paddw m1, m0, [t2+r10*2+416* 6] |
| pmulld m16, m9 ; -a3 * 9 |
| pmulld m17, m9 |
| mova [t2+r10*2+416* 6], m0 |
| mova [t2+r10*2+416* 8], m2 |
| mova [t2+r10*2+416*10], m3 |
| mova m2, [t1+r10*2+416*0] |
| mova m3, [t1+r10*2+416*2] |
| mova m18, [t1+r10*2+416*4] |
| punpcklwd m0, m4, m1 ; b3 |
| vpdpwssd m16, m0, m0 ; -p3 |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| pmulld m16, m12 ; p3 * s1 |
| pmulld m17, m12 |
| pmaddwd m0, m13 ; b3 * 455 |
| pmaddwd m1, m13 |
| mova [t3+r10*4+416*8+ 8], m2 |
| mova [t3+r10*4+416*0+ 8], m3 |
| mova [t3+r10*4+416*0+72], m18 |
| vpalignr m17{k2}, m16, m16, 2 |
| mova m16, m22 |
| paddusw m17, m14 |
| psraw m17, 4 ; min(z3, 255) - 256 |
| vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x3 |
| pandn m16, m24, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| paddw m2, m2 ; cc5 |
| paddd m3, m3 |
| paddd m18, m18 |
| mova [t1+r10*2+416*0], m2 |
| mova [t1+r10*2+416*2], m3 |
| mova [t1+r10*2+416*4], m18 |
| paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m15 |
| vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) |
| vpternlogd m17, m1, m24, 0xd8 |
| mova [t3+r10*4+416*4+ 8], m16 |
| mova [t3+r10*4+416*4+ 24], xm17 |
| vextracti32x4 [t3+r10*4+416*4+ 56], m17, 2 |
| mova [t3+r10*4+416*4+ 72], m17 |
| vextracti128 [t3+r10*4+416*4+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+416*4+104], m16, 3 |
| add r10, 32 |
| jl .v0_loop |
| ret |
| .v1: ; vertical boxsums + ab (odd rows) |
| lea r10, [wq-2] |
| .v1_loop: |
| mova m0, [t1+r10*2+416* 8] |
| paddd m16, m0, [t2+r10*2+416* 8] |
| mova m1, [t1+r10*2+416*10] |
| paddd m17, m1, [t2+r10*2+416*10] |
| mova m2, [t3+r10*4+416*0+ 8] |
| paddd m18, m2, [t2+r10*2+416* 2] |
| mova m3, [t3+r10*4+416*0+72] |
| paddd m19, m3, [t2+r10*2+416* 4] |
| paddd m18, [t1+r10*2+416* 2] |
| paddd m19, [t1+r10*2+416* 4] |
| mova [t2+r10*2+416* 8], m0 |
| mova [t2+r10*2+416*10], m1 |
| mova [t2+r10*2+416* 2], m2 |
| mova [t2+r10*2+416* 4], m3 |
| pmulld m16, m9 ; -a3 * 9 |
| pmulld m17, m9 |
| pmulld m18, m10 ; -a5 * 25 |
| pmulld m19, m10 |
| mova m0, [t1+r10*2+416* 6] |
| paddw m1, m0, [t2+r10*2+416* 6] |
| mova m2, [t3+r10*4+416*8+ 8] |
| paddw m3, m2, [t2+r10*2+416*0] |
| paddw m3, [t1+r10*2+416*0] |
| mova [t2+r10*2+416* 6], m0 |
| mova [t2+r10*2+416*0], m2 |
| punpcklwd m0, m4, m1 ; b3 |
| vpdpwssd m16, m0, m0 ; -p3 |
| punpckhwd m1, m4, m1 |
| vpdpwssd m17, m1, m1 |
| punpcklwd m2, m3, m4 ; b5 |
| vpdpwssd m18, m2, m2 ; -p5 |
| punpckhwd m3, m4 |
| vpdpwssd m19, m3, m3 |
| pmulld m16, m12 ; p3 * s1 |
| pmulld m17, m12 |
| pmulld m18, m11 ; p5 * s0 |
| pmulld m19, m11 |
| pmaddwd m0, m13 ; b3 * 455 |
| pmaddwd m1, m13 |
| pmaddwd m2, m13 ; b5 * 164 |
| pmaddwd m3, m13 |
| vpalignr m17{k2}, m16, m16, 2 |
| vpalignr m19{k2}, m18, m18, 2 |
| paddusw m17, m14 |
| mova m16, m22 |
| psraw m17, 4 ; min(z3, 255) - 256 |
| vpermt2b m16, m17, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k3, m17 |
| vpermi2b m17, m20, m21 ; sgr_x_by_x[ 0..127] |
| paddusw m19, m14 |
| mova m18, m22 |
| psraw m19, 4 ; min(z5, 255) - 256 |
| vpermt2b m18, m19, m23 ; sgr_x_by_x[128..255] |
| vpmovb2m k4, m19 |
| vpermi2b m19, m20, m21 ; sgr_x_by_x[ 0..127] |
| vmovdqu8 m17{k3}, m16 ; x3 |
| vmovdqu8 m19{k4}, m18 ; x5 |
| pandn m16, m24, m17 |
| psrld m17, 16 |
| pmulld m0, m16 |
| pmulld m1, m17 |
| pandn m18, m24, m19 |
| psrld m19, m19, 16 |
| pmulld m2, m18 |
| pmulld m3, m19 |
| paddd m0, m15 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) |
| paddd m1, m15 |
| vpternlogd m16, m0, m24, 0xd8 ; a3 | (b3 << 12) |
| vpternlogd m17, m1, m24, 0xd8 |
| mova [t3+r10*4+416*8+ 8], m16 |
| mova [t3+r10*4+416*8+ 24], xm17 |
| vextracti32x4 [t3+r10*4+416*8+ 56], m17, 2 |
| paddd m2, m15 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) |
| paddd m3, m15 |
| mova [t3+r10*4+416*8+ 72], m17 |
| vextracti128 [t3+r10*4+416*8+ 72], ym16, 1 |
| vextracti32x4 [t3+r10*4+416*8+104], m16, 3 |
| vpternlogd m18, m2, m24, 0xd8 ; a5 | (b5 << 12) |
| vpternlogd m19, m3, m24, 0xd8 |
| mova [t3+r10*4+416*0+ 8], m18 |
| mova [t3+r10*4+416*0+ 24], xm19 |
| vextracti32x4 [t3+r10*4+416*0+ 56], m19, 2 |
| mova [t3+r10*4+416*0+ 72], m19 |
| vextracti128 [t3+r10*4+416*0+ 72], ym18, 1 |
| vextracti32x4 [t3+r10*4+416*0+104], m18, 3 |
| add r10, 32 |
| jl .v1_loop |
| mov r10, t2 |
| mov t2, t1 |
| mov t1, r10 |
| ret |
| .prep_n: ; initial neighbor setup |
| mov r10, wq |
| .prep_n_loop: |
| movu m0, [t3+r10*4+416*0+4] |
| paddd m1, m0, [t3+r10*4+416*0+0] |
| mova m16, [t3+r10*4+416*4+0] |
| paddd m1, [t3+r10*4+416*0+8] |
| mova m17, [t3+r10*4+416*8+0] |
| paddd m16, [t3+r10*4+416*4+8] |
| paddd m17, [t3+r10*4+416*8+8] |
| paddd m2, m16, [t3+r10*4+416*4+4] |
| paddd m3, m17, [t3+r10*4+416*8+4] |
| paddd m0, m1 |
| pslld m1, 2 |
| pslld m2, 2 |
| paddd m1, m0 ; ab5 565 |
| paddd m3, m3 ; ab3[ 0] 222 |
| psubd m2, m16 ; ab3[-1] 343 |
| mova [t3+r10*4+416*20], m3 |
| pandn m0, m24, m1 ; a5 565 |
| mova [t3+r10*4+416*24], m2 |
| psrld m1, 12 ; b5 565 |
| mova [t3+r10*4+416*12], m0 |
| paddd m3, m3 |
| mova [t3+r10*4+416*16], m1 |
| psubd m3, m17 ; ab3[ 0] 343 |
| mova [t3+r10*4+416*28], m3 |
| add r10, 16 |
| jl .prep_n_loop |
| ret |
| ALIGN function_align |
| .n0: ; neighbor + output (even rows) |
| mov r10, wq |
| .n0_loop: |
| movu m2, [t3+r10*4+4] |
| paddd m3, m2, [t3+r10*4+0] |
| paddd m3, [t3+r10*4+8] |
| mova m1, [t3+r10*4+416*4+0] |
| paddd m2, m3 |
| pslld m3, 2 |
| paddd m1, [t3+r10*4+416*4+8] |
| paddd m3, m2 |
| pandn m2, m24, m3 |
| psrld m3, 12 |
| paddd m0, m2, [t3+r10*4+416*12] ; a5 |
| paddd m16, m3, [t3+r10*4+416*16] ; b5 + (1 << 8) |
| mova [t3+r10*4+416*12], m2 |
| mova [t3+r10*4+416*16], m3 |
| paddd m2, m1, [t3+r10*4+416*4+4] |
| paddd m2, m2 ; ab3[ 1] 222 |
| mova m3, [t3+r10*4+416*20] |
| paddd m17, m3, [t3+r10*4+416*24] ; ab3[ 0] 222 + ab3[-1] 343 |
| mova [t3+r10*4+416*20], m2 |
| paddd m2, m2 |
| psubd m2, m1 ; ab3[ 1] 343 |
| mova [t3+r10*4+416*24], m2 |
| paddd m2, m3 ; ab3[ 0] 222 + ab3[ 1] 343 |
| pandn m1, m24, m17 |
| psrld m17, 12 |
| pandn m3, m24, m2 |
| psrld m2, 12 |
| paddd m1, m3 ; a3 |
| pmovzxbd m3, [dstq+r10] |
| paddd m17, m2 ; b3 + (1 << 8) |
| pmaddwd m0, m3 ; a5 * src |
| pmaddwd m1, m3 ; a3 * src |
| vpshldd m3, m25, 16 ; (dst << 16) + (1 << 15) |
| psubd m16, m0 ; b5 - a5 * src + (1 << 8) |
| psubd m17, m1 ; b3 - a3 * src + (1 << 8) |
| psrld m16, 9 |
| pslld m17, 7 |
| vmovdqu8 m17{k2}, m16 |
| vpdpwssd m3, m17, m26 |
| packuswb m3, m2 |
| vpermb m16, m27, m3 |
| mova [dstq+r10], xm16 |
| add r10, 16 |
| jl .n0_loop |
| add dstq, strideq |
| ret |
| ALIGN function_align |
| .n1: ; neighbor + output (odd rows) |
| mov r10, wq |
| .n1_loop: |
| mova m1, [t3+r10*4+416*8+0] |
| paddd m1, [t3+r10*4+416*8+8] |
| paddd m2, m1, [t3+r10*4+416*8+4] |
| paddd m2, m2 ; ab3[ 1] 222 |
| mova m0, [t3+r10*4+416*20] |
| paddd m17, m0, [t3+r10*4+416*28] ; ab3[ 0] 222 + ab3[-1] 343 |
| pmovzxbd m3, [dstq+r10] |
| mova [t3+r10*4+416*20], m2 |
| paddd m2, m2 |
| psubd m2, m1 ; ab3[ 1] 343 |
| mova [t3+r10*4+416*28], m2 |
| paddd m0, m2 ; ab3[ 0] 222 + ab3[ 1] 343 |
| pandn m1, m24, m17 |
| psrld m17, 12 |
| pandn m2, m24, m0 |
| psrld m0, 12 |
| paddd m1, m2 ; a3 |
| paddd m17, m0 ; b3 + (1 << 8) |
| mova m16, [t3+r10*4+416*16] ; b5 + (1 << 7) |
| pmaddwd m1, m3 ; a3 * src |
| pmaddwd m0, m3, [t3+r10*4+416*12] ; a5 * src |
| vpshldd m3, m25, 16 ; (dst << 16) + (1 << 15) |
| psubd m17, m1 ; b3 - a3 * src + (1 << 8) |
| psubd m16, m0 ; b5 - a5 * src + (1 << 7) |
| pslld m17, 7 |
| palignr m17{k2}, m16, m16, 1 |
| vpdpwssd m3, m17, m26 |
| packuswb m3, m3 |
| vpermb m16, m27, m3 |
| mova [dstq+r10], xm16 |
| add r10, 16 |
| jl .n1_loop |
| add dstq, strideq |
| ret |
| |
| %endif ; ARCH_X86_64 |
