| ; Copyright © 2018, VideoLAN and dav1d authors |
| ; Copyright © 2018, Two Orioles, LLC |
| ; Copyright © 2018, VideoLabs |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "ext/x86/x86inc.asm" |
| |
| SECTION_RODATA 16 |
| |
| pb_right_ext_mask: times 16 db 0xff |
| times 16 db 0 |
| pb_14x0_1_2: times 14 db 0 |
| db 1, 2 |
| pb_0_to_15_min_n: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 13, 13 |
| db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 14 |
| pb_unpcklwdw: db 0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13 |
| pb_0: times 16 db 0 |
| pb_2: times 16 db 2 |
| pb_3: times 16 db 3 |
| pb_4: times 16 db 4 |
| pb_15: times 16 db 15 |
| pb_0_1: times 8 db 0, 1 |
| pb_6_7: times 8 db 6, 7 |
| pb_14_15: times 8 db 14, 15 |
| pw_1: times 8 dw 1 |
| pw_16: times 8 dw 16 |
| pw_128: times 8 dw 128 |
| pw_255: times 8 dw 255 |
| pw_256: times 8 dw 256 |
| pw_2048: times 8 dw 2048 |
| pw_16380: times 8 dw 16380 |
| pw_5_6: times 4 dw 5, 6 |
| pw_0_128: times 4 dw 0, 128 |
| pd_1024: times 4 dd 1024 |
| %if ARCH_X86_32 |
| pd_256: times 4 dd 256 |
| pd_512: times 4 dd 512 |
| pd_2048: times 4 dd 2048 |
| %endif |
| pd_0xF0080029: times 4 dd 0xF0080029 |
| pd_0xF00801C7: times 4 dd 0XF00801C7 |
| |
| cextern sgr_x_by_x |
| |
| SECTION .text |
| |
| %if ARCH_X86_32 |
| %define PIC_base_offset $$ |
| |
| %macro SETUP_PIC 1-3 1,0 ; PIC_reg, save_PIC_reg, restore_PIC_reg |
| %assign pic_reg_stk_off 4 |
| %xdefine PIC_reg %1 |
| %if %2 == 1 |
| mov [esp], %1 |
| %endif |
| LEA PIC_reg, PIC_base_offset |
| %if %3 == 1 |
| XCHG_PIC_REG |
| %endif |
| %endmacro |
| |
| %macro XCHG_PIC_REG 0 |
| mov [esp+pic_reg_stk_off], PIC_reg |
| %assign pic_reg_stk_off (pic_reg_stk_off+4) % 8 |
| mov PIC_reg, [esp+pic_reg_stk_off] |
| %endmacro |
| |
| %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) |
| |
| %else |
| %macro XCHG_PIC_REG 0 |
| %endmacro |
| |
| %define PIC_sym(sym) (sym) |
| %endif |
| |
| %macro PALIGNR 4 ; dst, src1, src2, shift |
| %if cpuflag(ssse3) |
| palignr %1, %2, %3, %4 |
| %else |
| %assign %%i regnumof%+%1 + 1 |
| %define %%tmp m %+ %%i |
| psrldq %1, %3, %4 |
| pslldq %%tmp, %2, 16-%4 |
| por %1, %%tmp |
| %endif |
| %endmacro |
| |
| %macro PMADDUBSW 5 ; dst, src, zero, tmp, reset_zero |
| %if cpuflag(ssse3) |
| pmaddubsw %1, %2 |
| %else |
| %if %5 == 1 |
| pxor %3, %3 |
| %endif |
| punpckhbw %4, %1, %3 |
| punpcklbw %1, %3 |
| pmaddwd %4, %2 |
| pmaddwd %1, %2 |
| packssdw %1, %4 |
| %endif |
| %endmacro |
| |
| ;;;;;;;;;;;;;;;;;;;;;; |
| ;; wiener ;; |
| ;;;;;;;;;;;;;;;;;;;;;; |
| |
| %macro WIENER_H 0 |
| %if ARCH_X86_64 |
| cglobal wiener_filter_h, 5, 15, 16, dst, left, src, stride, fh, w, h, edge |
| mov edged, edgem |
| movifnidn wd, wm |
| mov hd, hm |
| %else |
| cglobal wiener_filter_h, 5, 7, 8, -84, dst, left, src, stride, fh, w, h, edge |
| mov r5, edgem |
| mov [esp+12], r5 |
| mov wd, wm |
| mov hd, hm |
| SETUP_PIC hd |
| %define m15 m0 |
| %define m14 m1 |
| %define m13 m2 |
| %define m12 m3 |
| %endif |
| |
| movq m15, [fhq] |
| %if cpuflag(ssse3) |
| pshufb m12, m15, [PIC_sym(pb_6_7)] |
| pshufb m13, m15, [PIC_sym(pb_4)] |
| pshufb m14, m15, [PIC_sym(pb_2)] |
| pshufb m15, m15, [PIC_sym(pb_0)] |
| %else |
| pshuflw m12, m15, q3333 |
| punpcklbw m15, m15 |
| pshufhw m13, m15, q0000 |
| pshuflw m14, m15, q2222 |
| pshuflw m15, m15, q0000 |
| punpcklqdq m12, m12 |
| punpckhqdq m13, m13 |
| punpcklqdq m14, m14 |
| punpcklqdq m15, m15 |
| psraw m13, 8 |
| psraw m14, 8 |
| psraw m15, 8 |
| %endif |
| |
| %if ARCH_X86_64 |
| mova m11, [pw_2048] |
| mova m10, [pw_16380] |
| lea r11, [pb_right_ext_mask] |
| |
| DEFINE_ARGS dst, left, src, stride, x, w, h, edge, srcptr, dstptr, xlim |
| %else |
| %define m10 [PIC_sym(pw_16380)] |
| %define m11 [PIC_sym(pw_2048)] |
| %define m12 [esp+0x14] |
| %define m13 [esp+0x24] |
| %define m14 [esp+0x34] |
| %define m15 [esp+0x44] |
| mova m12, m3 |
| mova m13, m2 |
| mova m14, m1 |
| mova m15, m0 |
| |
| DEFINE_ARGS dst, left, src, stride, x, w, h, edge |
| %define srcptrq srcq |
| %define dstptrq dstq |
| %define hd dword [esp+ 0] |
| %define edged dword [esp+12] |
| %define xlimd dword [esp+16] |
| %endif |
| |
| ; if (edge & has_right) align_w_to_16 |
| ; else w -= 3, and use that as limit in x loop |
| test edged, 2 ; has_right |
| jnz .align |
| mov xlimd, -3 |
| jmp .loop |
| .align: |
| add wd, 15 |
| and wd, ~15 |
| %if ARCH_X86_64 |
| xor xlimd, xlimd |
| %else |
| mov xlimd, 0 |
| %endif |
| |
| ; main y loop for vertical filter |
| .loop: |
| %if ARCH_X86_64 |
| mov srcptrq, srcq |
| mov dstptrq, dstq |
| lea xd, [wq+xlimq] |
| %else |
| mov [esp+8], srcq |
| mov [esp+4], dstq |
| mov xd, xlimd |
| add xd, wd |
| %endif |
| |
| ; load left edge pixels |
| test edged, 1 ; have_left |
| jz .emu_left |
| test leftq, leftq ; left == NULL for the edge-extended bottom/top |
| jz .load_left_combined |
| movd m0, [leftq] |
| movd m1, [srcq] |
| punpckldq m0, m1 |
| pslldq m0, 9 |
| add leftq, 4 |
| jmp .left_load_done |
| .load_left_combined: |
| movq m0, [srcq-3] |
| pslldq m0, 10 |
| jmp .left_load_done |
| .emu_left: |
| movd m0, [srcq] |
| %if cpuflag(ssse3) |
| pshufb m0, [PIC_sym(pb_14x0_1_2)] |
| %else |
| pslldq m1, m0, 13 |
| punpcklbw m0, m0 |
| pshuflw m0, m0, q0000 |
| punpcklqdq m0, m0 |
| psrldq m0, 2 |
| por m0, m1 |
| %endif |
| |
| ; load right edge pixels |
| .left_load_done: |
| cmp xd, 16 |
| jg .main_load |
| test xd, xd |
| jg .load_and_splat |
| je .splat_right |
| |
| ; for very small images (w=[1-2]), edge-extend the original cache, |
| ; ugly, but only runs in very odd cases |
| %if cpuflag(ssse3) |
| add wd, wd |
| %if ARCH_X86_64 |
| pshufb m0, [r11-pb_right_ext_mask+pb_0_to_15_min_n+wq*8-16] |
| %else |
| pshufb m0, [PIC_sym(pb_0_to_15_min_n)+wq*8-16] |
| %endif |
| shr wd, 1 |
| %else |
| shl wd, 4 |
| pcmpeqd m2, m2 |
| movd m3, wd |
| psrldq m2, 2 |
| punpckhbw m1, m0, m0 |
| pshufhw m1, m1, q1122 |
| psllq m1, m3 |
| pand m0, m2 |
| pandn m2, m1 |
| por m0, m2 |
| shr wd, 4 |
| %endif |
| |
| ; main x loop, mostly this starts in .main_load |
| .splat_right: |
| ; no need to load new pixels, just extend them from the (possibly previously |
| ; extended) previous load into m0 |
| %if cpuflag(ssse3) |
| pshufb m1, m0, [PIC_sym(pb_15)] |
| %else |
| punpckhbw m1, m0, m0 |
| pshufhw m1, m1, q3333 |
| punpckhqdq m1, m1 |
| %endif |
| jmp .main_loop |
| .load_and_splat: |
| ; load new pixels and extend edge for right-most |
| movu m1, [srcptrq+3] |
| %if ARCH_X86_64 |
| sub r11, xq |
| movu m2, [r11+16] |
| add r11, xq |
| %else |
| sub PIC_reg, xd |
| movu m2, [PIC_sym(pb_right_ext_mask)+16] |
| add PIC_reg, xd |
| %endif |
| movd m3, [srcptrq+2+xq] |
| %if cpuflag(ssse3) |
| pshufb m3, [PIC_sym(pb_0)] |
| %else |
| punpcklbw m3, m3 |
| pshuflw m3, m3, q0000 |
| punpcklqdq m3, m3 |
| %endif |
| pand m1, m2 |
| pxor m2, [PIC_sym(pb_right_ext_mask)] |
| pand m3, m2 |
| pxor m2, [PIC_sym(pb_right_ext_mask)] |
| por m1, m3 |
| jmp .main_loop |
| .main_load: |
| ; load subsequent line |
| movu m1, [srcptrq+3] |
| .main_loop: |
| %if ARCH_X86_64 |
| PALIGNR m2, m1, m0, 10 |
| PALIGNR m3, m1, m0, 11 |
| PALIGNR m4, m1, m0, 12 |
| PALIGNR m5, m1, m0, 13 |
| PALIGNR m6, m1, m0, 14 |
| PALIGNR m7, m1, m0, 15 |
| |
| punpcklbw m0, m2, m1 |
| punpckhbw m2, m1 |
| punpcklbw m8, m3, m7 |
| punpckhbw m3, m7 |
| punpcklbw m7, m4, m6 |
| punpckhbw m4, m6 |
| PMADDUBSW m0, m15, m6, m9, 1 |
| PMADDUBSW m2, m15, m6, m9, 0 |
| PMADDUBSW m8, m14, m6, m9, 0 |
| PMADDUBSW m3, m14, m6, m9, 0 |
| PMADDUBSW m7, m13, m6, m9, 0 |
| PMADDUBSW m4, m13, m6, m9, 0 |
| paddw m0, m8 |
| paddw m2, m3 |
| %if cpuflag(ssse3) |
| pxor m6, m6 |
| %endif |
| punpcklbw m3, m5, m6 |
| punpckhbw m5, m6 |
| psllw m8, m3, 7 |
| psllw m6, m5, 7 |
| psubw m8, m10 |
| psubw m6, m10 |
| pmullw m3, m12 |
| pmullw m5, m12 |
| paddw m0, m7 |
| paddw m2, m4 |
| paddw m0, m3 |
| paddw m2, m5 |
| paddsw m0, m8 |
| paddsw m2, m6 |
| psraw m0, 3 |
| psraw m2, 3 |
| paddw m0, m11 |
| paddw m2, m11 |
| mova [dstptrq+ 0], m0 |
| mova [dstptrq+16], m2 |
| %else |
| PALIGNR m2, m1, m0, 10 |
| punpcklbw m3, m2, m1 |
| punpckhbw m2, m1 |
| PMADDUBSW m3, m15, m4, m5, 1 |
| PMADDUBSW m2, m15, m4, m5, 0 |
| PALIGNR m4, m1, m0, 11 |
| PALIGNR m5, m1, m0, 15 |
| punpcklbw m6, m4, m5 |
| punpckhbw m4, m5 |
| PMADDUBSW m6, m14, m5, m7, 1 |
| PMADDUBSW m4, m14, m5, m7, 0 |
| paddw m3, m6 |
| paddw m2, m4 |
| PALIGNR m4, m1, m0, 12 |
| PALIGNR m5, m1, m0, 14 |
| punpcklbw m6, m4, m5 |
| punpckhbw m4, m5 |
| PMADDUBSW m6, m13, m5, m7, 1 |
| PMADDUBSW m4, m13, m5, m7, 0 |
| paddw m3, m6 |
| paddw m2, m4 |
| PALIGNR m6, m1, m0, 13 |
| %if cpuflag(ssse3) |
| pxor m5, m5 |
| %endif |
| punpcklbw m4, m6, m5 |
| punpckhbw m6, m5 |
| psllw m5, m4, 7 |
| psllw m7, m6, 7 |
| psubw m5, m10 |
| psubw m7, m10 |
| pmullw m4, m12 |
| pmullw m6, m12 |
| paddw m3, m4 |
| paddw m2, m6 |
| paddsw m3, m5 |
| paddsw m2, m7 |
| psraw m3, 3 |
| psraw m2, 3 |
| paddw m3, m11 |
| paddw m2, m11 |
| mova [dstptrq+ 0], m3 |
| mova [dstptrq+16], m2 |
| %endif |
| |
| mova m0, m1 |
| add srcptrq, 16 |
| add dstptrq, 32 |
| sub xd, 16 |
| cmp xd, 16 |
| jg .main_load |
| test xd, xd |
| jg .load_and_splat |
| cmp xd, xlimd |
| jg .splat_right |
| |
| %if ARCH_X86_32 |
| mov srcq, [esp+8] |
| mov dstq, [esp+4] |
| %endif |
| add srcq, strideq |
| add dstq, 384*2 |
| dec hd |
| jg .loop |
| RET |
| %endmacro |
| |
| %macro WIENER_V 0 |
| %if ARCH_X86_64 |
| cglobal wiener_filter_v, 4, 10, 16, dst, stride, mid, w, h, fv, edge |
| mov edged, edgem |
| movifnidn fvq, fvmp |
| movifnidn hd, hm |
| movq m15, [fvq] |
| pshufd m14, m15, q1111 |
| pshufd m15, m15, q0000 |
| paddw m14, [pw_0_128] |
| mova m12, [pd_1024] |
| |
| DEFINE_ARGS dst, stride, mid, w, h, y, edge, ylim, mptr, dstptr |
| |
| mov ylimd, edged |
| and ylimd, 8 ; have_bottom |
| shr ylimd, 2 |
| sub ylimd, 3 |
| %else |
| cglobal wiener_filter_v, 5, 7, 8, -96, dst, stride, mid, w, h, fv, edge |
| %define ylimd [esp+12] |
| |
| mov r5d, edgem |
| and r5d, 8 |
| shr r5d, 2 |
| sub r5d, 3 |
| mov ylimd, r5d |
| mov fvq, fvmp |
| mov edged, edgem |
| |
| SETUP_PIC edged |
| |
| movq m0, [fvq] |
| pshufd m1, m0, q1111 |
| pshufd m0, m0, q0000 |
| paddw m1, [PIC_sym(pw_0_128)] |
| mova [esp+0x50], m0 |
| mova [esp+0x40], m1 |
| |
| DEFINE_ARGS dst, stride, mid, w, h, y, edge |
| %define mptrq midq |
| %define dstptrq dstq |
| %define edged dword [esp] |
| %endif |
| |
| ; main x loop for vertical filter, does one column of 16 pixels |
| .loop_x: |
| mova m3, [midq] ; middle line |
| |
| ; load top pixels |
| test edged, 4 ; have_top |
| jz .emu_top |
| mova m0, [midq-384*4] |
| mova m2, [midq-384*2] |
| mova m1, m0 |
| jmp .load_bottom_pixels |
| .emu_top: |
| mova m0, m3 |
| mova m1, m3 |
| mova m2, m3 |
| |
| ; load bottom pixels |
| .load_bottom_pixels: |
| mov yd, hd |
| %if ARCH_X86_64 |
| mov mptrq, midq |
| mov dstptrq, dstq |
| add yd, ylimd |
| %else |
| mov [esp+8], midq |
| mov [esp+4], dstq |
| add yd, ylimd |
| %endif |
| jg .load_threelines |
| |
| ; the remainder here is somewhat messy but only runs in very weird |
| ; circumstances at the bottom of the image in very small blocks (h=[1-3]), |
| ; so performance is not terribly important here... |
| je .load_twolines |
| cmp yd, -1 |
| je .load_oneline |
| ; h == 1 case |
| mova m5, m3 |
| mova m4, m3 |
| mova m6, m3 |
| jmp .loop |
| .load_oneline: |
| ; h == 2 case |
| mova m4, [midq+384*2] |
| mova m5, m4 |
| mova m6, m4 |
| jmp .loop |
| .load_twolines: |
| ; h == 3 case |
| mova m4, [midq+384*2] |
| mova m5, [midq+384*4] |
| mova m6, m5 |
| jmp .loop |
| .load_threelines: |
| ; h > 3 case |
| mova m4, [midq+384*2] |
| mova m5, [midq+384*4] |
| ; third line loaded in main loop below |
| |
| ; main y loop for vertical filter |
| .loop_load: |
| ; load one line into m6. if that pixel is no longer available, do |
| ; nothing, since m6 still has the data from the previous line in it. We |
| ; try to structure the loop so that the common case is evaluated fastest |
| mova m6, [mptrq+384*6] |
| .loop: |
| %if ARCH_X86_64 |
| paddw m7, m0, m6 |
| paddw m8, m1, m5 |
| paddw m9, m2, m4 |
| punpcklwd m10, m7, m8 |
| punpckhwd m7, m8 |
| punpcklwd m11, m9, m3 |
| punpckhwd m9, m3 |
| pmaddwd m10, m15 |
| pmaddwd m7, m15 |
| pmaddwd m11, m14 |
| pmaddwd m9, m14 |
| paddd m10, m12 |
| paddd m7, m12 |
| paddd m10, m11 |
| paddd m7, m9 |
| psrad m10, 11 |
| psrad m7, 11 |
| packssdw m10, m7 |
| packuswb m10, m10 |
| movq [dstptrq], m10 |
| %else |
| mova [esp+0x30], m1 |
| mova [esp+0x20], m2 |
| mova [esp+0x10], m3 |
| paddw m0, m6 |
| paddw m1, m5 |
| paddw m2, m4 |
| punpcklwd m7, m2, m3 |
| punpckhwd m2, m3 |
| punpcklwd m3, m0, m1 |
| punpckhwd m0, m1 |
| mova m1, [esp+0x50] |
| pmaddwd m3, m1 |
| pmaddwd m0, m1 |
| mova m1, [esp+0x40] |
| pmaddwd m7, m1 |
| pmaddwd m2, m1 |
| paddd m3, [PIC_sym(pd_1024)] |
| paddd m0, [PIC_sym(pd_1024)] |
| paddd m3, m7 |
| paddd m0, m2 |
| psrad m3, 11 |
| psrad m0, 11 |
| packssdw m3, m0 |
| packuswb m3, m3 |
| movq [dstq], m3 |
| mova m1, [esp+0x30] |
| mova m2, [esp+0x20] |
| mova m3, [esp+0x10] |
| %endif |
| ; shift pixels one position |
| mova m0, m1 |
| mova m1, m2 |
| mova m2, m3 |
| mova m3, m4 |
| mova m4, m5 |
| mova m5, m6 |
| add dstptrq, strideq |
| add mptrq, 384*2 |
| dec yd |
| jg .loop_load |
| ; for the bottom pixels, continue using m6 (as extended edge) |
| cmp yd, ylimd |
| jg .loop |
| |
| %if ARCH_X86_32 |
| mov midq, [esp+8] |
| mov dstq, [esp+4] |
| %endif |
| add dstq, 8 |
| add midq, 16 |
| sub wd, 8 |
| jg .loop_x |
| RET |
| %endmacro |
| |
| INIT_XMM sse2 |
| WIENER_H |
| WIENER_V |
| |
| INIT_XMM ssse3 |
| WIENER_H |
| WIENER_V |
| |
| ;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ;; self-guided ;; |
| ;;;;;;;;;;;;;;;;;;;;;;;;;; |
| |
| %macro MULLD 2 |
| pmulhuw m5, %1, %2 |
| pmullw %1, %2 |
| pslld m5, 16 |
| paddd %1, m5 |
| %endmacro |
| |
| %macro GATHERDD 2 |
| mova m5, m7 |
| movd r6d, %2 |
| %if ARCH_X86_64 |
| movd %1, [r5+r6] |
| pextrw r6d, %2, 2 |
| pinsrw m5, [r5+r6+2], 3 |
| pextrw r6d, %2, 4 |
| pinsrw %1, [r5+r6+2], 5 |
| pextrw r6d, %2, 6 |
| pinsrw m5, [r5+r6+2], 7 |
| %else |
| movd %1, [PIC_sym(sgr_x_by_x-0xF03)+r6] |
| pextrw r6d, %2, 2 |
| pinsrw m5, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 3 |
| pextrw r6d, %2, 4 |
| pinsrw %1, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 5 |
| pextrw r6d, %2, 6 |
| pinsrw m5, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 7 |
| %endif |
| por %1, m5 |
| %endmacro |
| |
| %if ARCH_X86_64 |
| cglobal sgr_box3_h, 5, 11, 8, sumsq, sum, left, src, stride, x, h, edge, w, xlim |
| mov xlimd, edgem |
| movifnidn xd, xm |
| mov hd, hm |
| mov edged, xlimd |
| and xlimd, 2 ; have_right |
| add xd, xlimd |
| xor xlimd, 2 ; 2*!have_right |
| %else |
| cglobal sgr_box3_h, 6, 7, 8, sumsq, sum, left, src, stride, x, h, edge, w, xlim |
| %define wq r0m |
| %define xlimd r1m |
| %define hd hmp |
| %define edged edgemp |
| |
| mov r6, edgem |
| and r6, 2 ; have_right |
| add xd, r6 |
| xor r6, 2 ; 2*!have_right |
| mov xlimd, r6 |
| SETUP_PIC r6, 0 |
| %endif |
| |
| jnz .no_right |
| add xd, 7 |
| and xd, ~7 |
| .no_right: |
| pxor m1, m1 |
| lea srcq, [srcq+xq] |
| lea sumq, [sumq+xq*2-2] |
| lea sumsqq, [sumsqq+xq*4-4] |
| neg xq |
| mov wq, xq |
| %if ARCH_X86_64 |
| lea r10, [pb_right_ext_mask+16] |
| %endif |
| .loop_y: |
| mov xq, wq |
| |
| ; load left |
| test edged, 1 ; have_left |
| jz .no_left |
| test leftq, leftq |
| jz .load_left_from_main |
| movd m0, [leftq] |
| pslldq m0, 12 |
| add leftq, 4 |
| jmp .expand_x |
| .no_left: |
| movd m0, [srcq+xq] |
| pshufb m0, [PIC_sym(pb_0)] |
| jmp .expand_x |
| .load_left_from_main: |
| movd m0, [srcq+xq-2] |
| pslldq m0, 14 |
| .expand_x: |
| punpckhbw xm0, xm1 |
| |
| ; when we reach this, m0 contains left two px in highest words |
| cmp xq, -8 |
| jle .loop_x |
| .partial_load_and_extend: |
| movd m3, [srcq-4] |
| pshufb m3, [PIC_sym(pb_3)] |
| movq m2, [srcq+xq] |
| punpcklbw m2, m1 |
| punpcklbw m3, m1 |
| %if ARCH_X86_64 |
| movu m4, [r10+xq*2] |
| %else |
| movu m4, [PIC_sym(pb_right_ext_mask+16)+xd*2] |
| %endif |
| pand m2, m4 |
| pandn m4, m3 |
| por m2, m4 |
| jmp .loop_x_noload |
| .right_extend: |
| pshufb m2, m0, [PIC_sym(pb_14_15)] |
| jmp .loop_x_noload |
| |
| .loop_x: |
| movq m2, [srcq+xq] |
| punpcklbw m2, m1 |
| .loop_x_noload: |
| palignr m3, m2, m0, 12 |
| palignr m4, m2, m0, 14 |
| |
| punpcklwd m5, m3, m2 |
| punpckhwd m6, m3, m2 |
| paddw m3, m4 |
| punpcklwd m7, m4, m1 |
| punpckhwd m4, m1 |
| pmaddwd m5, m5 |
| pmaddwd m6, m6 |
| pmaddwd m7, m7 |
| pmaddwd m4, m4 |
| paddd m5, m7 |
| paddd m6, m4 |
| paddw m3, m2 |
| movu [sumq+xq*2], m3 |
| movu [sumsqq+xq*4+ 0], m5 |
| movu [sumsqq+xq*4+16], m6 |
| |
| mova m0, m2 |
| add xq, 8 |
| |
| ; if x <= -8 we can reload more pixels |
| ; else if x < 0 we reload and extend (this implies have_right=0) |
| ; else if x < xlimd we extend from previous load (this implies have_right=0) |
| ; else we are done |
| |
| cmp xd, -8 |
| jle .loop_x |
| test xd, xd |
| jl .partial_load_and_extend |
| cmp xd, xlimd |
| jl .right_extend |
| |
| add sumsqq, (384+16)*4 |
| add sumq, (384+16)*2 |
| add srcq, strideq |
| dec hd |
| jg .loop_y |
| RET |
| |
| %if ARCH_X86_64 |
| cglobal sgr_box3_v, 4, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_base, sum_base, ylim |
| movifnidn edged, edgem |
| %else |
| cglobal sgr_box3_v, 5, 7, 8, -28, sumsq, sum, w, h, edge, x, y |
| %define sumsq_baseq dword [esp+0] |
| %define sum_baseq dword [esp+4] |
| %define ylimd dword [esp+8] |
| %define m8 [esp+12] |
| %endif |
| mov xq, -2 |
| %if ARCH_X86_64 |
| mov ylimd, edged |
| and ylimd, 8 ; have_bottom |
| shr ylimd, 2 |
| sub ylimd, 2 ; -2 if have_bottom=0, else 0 |
| mov sumsq_baseq, sumsqq |
| mov sum_baseq, sumq |
| .loop_x: |
| mov sumsqq, sumsq_baseq |
| mov sumq, sum_baseq |
| lea yd, [hd+ylimd+2] |
| %else |
| mov yd, edged |
| and yd, 8 ; have_bottom |
| shr yd, 2 |
| sub yd, 2 ; -2 if have_bottom=0, else 0 |
| mov sumsq_baseq, sumsqq |
| mov sum_baseq, sumq |
| mov ylimd, yd |
| .loop_x: |
| mov sumsqd, sumsq_baseq |
| mov sumd, sum_baseq |
| lea yd, [hd+2] |
| add yd, ylimd |
| %endif |
| lea sumsqq, [sumsqq+xq*4+4-(384+16)*4] |
| lea sumq, [sumq+xq*2+2-(384+16)*2] |
| test edged, 4 ; have_top |
| jnz .load_top |
| movu m0, [sumsqq+(384+16)*4*1] |
| movu m1, [sumsqq+(384+16)*4*1+16] |
| mova m2, m0 |
| mova m3, m1 |
| mova m4, m0 |
| mova m5, m1 |
| movu m6, [sumq+(384+16)*2*1] |
| mova m7, m6 |
| mova m8, m6 |
| jmp .loop_y_noload |
| .load_top: |
| movu m0, [sumsqq-(384+16)*4*1] ; l2sq [left] |
| movu m1, [sumsqq-(384+16)*4*1+16] ; l2sq [right] |
| movu m2, [sumsqq-(384+16)*4*0] ; l1sq [left] |
| movu m3, [sumsqq-(384+16)*4*0+16] ; l1sq [right] |
| movu m6, [sumq-(384+16)*2*1] ; l2 |
| movu m7, [sumq-(384+16)*2*0] ; l1 |
| .loop_y: |
| %if ARCH_X86_64 |
| movu m8, [sumq+(384+16)*2*1] ; l0 |
| %else |
| movu m4, [sumq+(384+16)*2*1] ; l0 |
| mova m8, m4 |
| %endif |
| movu m4, [sumsqq+(384+16)*4*1] ; l0sq [left] |
| movu m5, [sumsqq+(384+16)*4*1+16] ; l0sq [right] |
| .loop_y_noload: |
| paddd m0, m2 |
| paddd m1, m3 |
| paddw m6, m7 |
| paddd m0, m4 |
| paddd m1, m5 |
| paddw m6, m8 |
| movu [sumsqq+ 0], m0 |
| movu [sumsqq+16], m1 |
| movu [sumq], m6 |
| |
| ; shift position down by one |
| mova m0, m2 |
| mova m1, m3 |
| mova m2, m4 |
| mova m3, m5 |
| mova m6, m7 |
| mova m7, m8 |
| add sumsqq, (384+16)*4 |
| add sumq, (384+16)*2 |
| dec yd |
| jg .loop_y |
| cmp yd, ylimd |
| jg .loop_y_noload |
| add xd, 8 |
| cmp xd, wd |
| jl .loop_x |
| RET |
| |
| cglobal sgr_calc_ab1, 4, 7, 12, a, b, w, h, s |
| movifnidn sd, sm |
| sub aq, (384+16-1)*4 |
| sub bq, (384+16-1)*2 |
| add hd, 2 |
| %if ARCH_X86_64 |
| LEA r5, sgr_x_by_x-0xF03 |
| %else |
| SETUP_PIC r5, 0 |
| %endif |
| movd m6, sd |
| pshuflw m6, m6, q0000 |
| punpcklqdq m6, m6 |
| pxor m7, m7 |
| DEFINE_ARGS a, b, w, h, x |
| %if ARCH_X86_64 |
| mova m8, [pd_0xF00801C7] |
| mova m9, [pw_256] |
| psrld m10, m9, 13 ; pd_2048 |
| mova m11, [pb_unpcklwdw] |
| %else |
| %define m8 [PIC_sym(pd_0xF00801C7)] |
| %define m9 [PIC_sym(pw_256)] |
| %define m10 [PIC_sym(pd_2048)] |
| %define m11 [PIC_sym(pb_unpcklwdw)] |
| %endif |
| .loop_y: |
| mov xq, -2 |
| .loop_x: |
| movq m0, [bq+xq*2] |
| movq m1, [bq+xq*2+(384+16)*2] |
| punpcklwd m0, m7 |
| punpcklwd m1, m7 |
| movu m2, [aq+xq*4] |
| movu m3, [aq+xq*4+(384+16)*4] |
| pslld m4, m2, 3 |
| pslld m5, m3, 3 |
| paddd m2, m4 ; aa * 9 |
| paddd m3, m5 |
| pmaddwd m4, m0, m0 |
| pmaddwd m5, m1, m1 |
| pmaddwd m0, m8 |
| pmaddwd m1, m8 |
| psubd m2, m4 ; p = aa * 9 - bb * bb |
| psubd m3, m5 |
| MULLD m2, m6 |
| MULLD m3, m6 |
| paddusw m2, m8 |
| paddusw m3, m8 |
| psrld m2, 20 ; z |
| psrld m3, 20 |
| GATHERDD m4, m2 ; xx |
| GATHERDD m2, m3 |
| psrld m4, 24 |
| psrld m2, 24 |
| packssdw m3, m4, m2 |
| pshufb m4, m11 |
| MULLD m0, m4 |
| pshufb m2, m11 |
| MULLD m1, m2 |
| psubw m5, m9, m3 |
| paddd m0, m10 |
| paddd m1, m10 |
| psrld m0, 12 |
| psrld m1, 12 |
| movq [bq+xq*2], m5 |
| psrldq m5, 8 |
| movq [bq+xq*2+(384+16)*2], m5 |
| movu [aq+xq*4], m0 |
| movu [aq+xq*4+(384+16)*4], m1 |
| add xd, 4 |
| cmp xd, wd |
| jl .loop_x |
| add aq, (384+16)*4*2 |
| add bq, (384+16)*2*2 |
| sub hd, 2 |
| jg .loop_y |
| RET |
| |
| %if ARCH_X86_64 |
| cglobal sgr_finish_filter1, 5, 13, 16, t, src, stride, a, b, w, h, \ |
| tmp_base, src_base, a_base, b_base, x, y |
| movifnidn wd, wm |
| mov hd, hm |
| mova m15, [pw_16] |
| mov tmp_baseq, tq |
| mov src_baseq, srcq |
| mov a_baseq, aq |
| mov b_baseq, bq |
| xor xd, xd |
| %else |
| cglobal sgr_finish_filter1, 7, 7, 8, -144, t, src, stride, a, b, x, y |
| %define tmp_baseq [esp+8] |
| %define src_baseq [esp+12] |
| %define a_baseq [esp+16] |
| %define b_baseq [esp+20] |
| %define wd [esp+24] |
| %define hd [esp+28] |
| mov tmp_baseq, tq |
| mov src_baseq, srcq |
| mov a_baseq, aq |
| mov b_baseq, bq |
| mov wd, xd |
| mov hd, yd |
| xor xd, xd |
| SETUP_PIC yd, 1, 1 |
| jmp .loop_start |
| %endif |
| |
| .loop_x: |
| mov tq, tmp_baseq |
| mov srcq, src_baseq |
| mov aq, a_baseq |
| mov bq, b_baseq |
| %if ARCH_X86_32 |
| .loop_start: |
| movu m0, [bq+xq*2-(384+16)*2-2] |
| movu m2, [bq+xq*2-(384+16)*2+2] |
| mova m1, [bq+xq*2-(384+16)*2] ; b:top |
| paddw m0, m2 ; b:tl+tr |
| movu m2, [bq+xq*2-2] |
| movu m3, [bq+xq*2+2] |
| paddw m1, [bq+xq*2] ; b:top+ctr |
| paddw m2, m3 ; b:l+r |
| mova [esp+0x80], m0 |
| mova [esp+0x70], m1 |
| mova [esp+0x60], m2 |
| %endif |
| movu m0, [aq+xq*4-(384+16)*4-4] |
| movu m2, [aq+xq*4-(384+16)*4+4] |
| mova m1, [aq+xq*4-(384+16)*4] ; a:top [first half] |
| paddd m0, m2 ; a:tl+tr [first half] |
| movu m2, [aq+xq*4-(384+16)*4-4+16] |
| movu m4, [aq+xq*4-(384+16)*4+4+16] |
| mova m3, [aq+xq*4-(384+16)*4+16] ; a:top [second half] |
| paddd m2, m4 ; a:tl+tr [second half] |
| movu m4, [aq+xq*4-4] |
| movu m5, [aq+xq*4+4] |
| paddd m1, [aq+xq*4] ; a:top+ctr [first half] |
| paddd m4, m5 ; a:l+r [first half] |
| movu m5, [aq+xq*4+16-4] |
| movu m6, [aq+xq*4+16+4] |
| paddd m3, [aq+xq*4+16] ; a:top+ctr [second half] |
| paddd m5, m6 ; a:l+r [second half] |
| %if ARCH_X86_64 |
| movu m6, [bq+xq*2-(384+16)*2-2] |
| movu m8, [bq+xq*2-(384+16)*2+2] |
| mova m7, [bq+xq*2-(384+16)*2] ; b:top |
| paddw m6, m8 ; b:tl+tr |
| movu m8, [bq+xq*2-2] |
| movu m9, [bq+xq*2+2] |
| paddw m7, [bq+xq*2] ; b:top+ctr |
| paddw m8, m9 ; b:l+r |
| %endif |
| |
| lea tq, [tq+xq*2] |
| lea srcq, [srcq+xq*1] |
| lea aq, [aq+xq*4+(384+16)*4] |
| lea bq, [bq+xq*2+(384+16)*2] |
| mov yd, hd |
| .loop_y: |
| %if ARCH_X86_64 |
| movu m9, [bq-2] |
| movu m10, [bq+2] |
| paddw m7, [bq] ; b:top+ctr+bottom |
| paddw m9, m10 ; b:bl+br |
| paddw m10, m7, m8 ; b:top+ctr+bottom+l+r |
| paddw m6, m9 ; b:tl+tr+bl+br |
| psubw m7, [bq-(384+16)*2*2] ; b:ctr+bottom |
| paddw m10, m6 |
| psllw m10, 2 |
| psubw m10, m6 ; aa |
| pxor m14, m14 |
| movq m12, [srcq] |
| punpcklbw m12, m14 |
| punpcklwd m6, m10, m15 |
| punpckhwd m10, m15 |
| punpcklwd m13, m12, m15 |
| punpckhwd m12, m15 |
| pmaddwd m6, m13 ; aa*src[x]+256 [first half] |
| pmaddwd m10, m12 ; aa*src[x]+256 [second half] |
| %else |
| paddd m1, [aq] ; a:top+ctr+bottom [first half] |
| paddd m3, [aq+16] ; a:top+ctr+bottom [second half] |
| mova [esp+0x50], m1 |
| mova [esp+0x40], m3 |
| mova [esp+0x30], m4 |
| movu m6, [aq-4] |
| movu m7, [aq+4] |
| paddd m1, m4 ; a:top+ctr+bottom+l+r [first half] |
| paddd m3, m5 ; a:top+ctr+bottom+l+r [second half] |
| paddd m6, m7 ; a:bl+br [first half] |
| movu m7, [aq+16-4] |
| movu m4, [aq+16+4] |
| paddd m7, m4 ; a:bl+br [second half] |
| paddd m0, m6 ; a:tl+tr+bl+br [first half] |
| paddd m2, m7 ; a:tl+tr+bl+br [second half] |
| paddd m1, m0 |
| paddd m3, m2 |
| pslld m1, 2 |
| pslld m3, 2 |
| psubd m1, m0 ; bb [first half] |
| psubd m3, m2 ; bb [second half] |
| %endif |
| |
| %if ARCH_X86_64 |
| movu m11, [aq-4] |
| movu m12, [aq+4] |
| paddd m1, [aq] ; a:top+ctr+bottom [first half] |
| paddd m11, m12 ; a:bl+br [first half] |
| movu m12, [aq+16-4] |
| movu m13, [aq+16+4] |
| paddd m3, [aq+16] ; a:top+ctr+bottom [second half] |
| paddd m12, m13 ; a:bl+br [second half] |
| paddd m13, m1, m4 ; a:top+ctr+bottom+l+r [first half] |
| paddd m14, m3, m5 ; a:top+ctr+bottom+l+r [second half] |
| paddd m0, m11 ; a:tl+tr+bl+br [first half] |
| paddd m2, m12 ; a:tl+tr+bl+br [second half] |
| paddd m13, m0 |
| paddd m14, m2 |
| pslld m13, 2 |
| pslld m14, 2 |
| psubd m13, m0 ; bb [first half] |
| psubd m14, m2 ; bb [second half] |
| psubd m1, [aq-(384+16)*4*2] ; a:ctr+bottom [first half] |
| psubd m3, [aq-(384+16)*4*2+16] ; a:ctr+bottom [second half] |
| %else |
| mova m4, [esp+0x80] |
| mova [esp+0x80], m5 |
| mova m5, [esp+0x70] |
| mova [esp+0x70], m6 |
| mova m6, [esp+0x60] |
| mova [esp+0x60], m7 |
| mova [esp+0x20], m1 |
| movu m7, [bq-2] |
| movu m1, [bq+2] |
| paddw m5, [bq] ; b:top+ctr+bottom |
| paddw m7, m1 |
| paddw m1, m5, m6 ; b:top+ctr+bottom+l+r |
| paddw m4, m7 ; b:tl+tr+bl+br |
| psubw m5, [bq-(384+16)*2*2] ; b:ctr+bottom |
| paddw m1, m4 |
| psllw m1, 2 |
| psubw m1, m4 ; aa |
| movq m0, [srcq] |
| XCHG_PIC_REG |
| punpcklbw m0, [PIC_sym(pb_right_ext_mask)+16] |
| punpcklwd m4, m1, [PIC_sym(pw_16)] |
| punpckhwd m1, [PIC_sym(pw_16)] |
| punpcklwd m2, m0, [PIC_sym(pw_16)] |
| punpckhwd m0, [PIC_sym(pw_16)] |
| XCHG_PIC_REG |
| pmaddwd m4, m2 ; aa*src[x]+256 [first half] |
| pmaddwd m1, m0 ; aa*src[x]+256 [second half] |
| %endif |
| |
| %if ARCH_X86_64 |
| paddd m6, m13 |
| paddd m10, m14 |
| psrad m6, 9 |
| psrad m10, 9 |
| packssdw m6, m10 |
| mova [tq], m6 |
| %else |
| paddd m4, [esp+0x20] |
| paddd m1, m3 |
| psrad m4, 9 |
| psrad m1, 9 |
| packssdw m4, m1 |
| mova [tq], m4 |
| %endif |
| |
| ; shift to next row |
| %if ARCH_X86_64 |
| mova m0, m4 |
| mova m2, m5 |
| mova m4, m11 |
| mova m5, m12 |
| mova m6, m8 |
| mova m8, m9 |
| %else |
| mova m1, [esp+0x50] |
| mova m3, [esp+0x40] |
| mova m0, [esp+0x30] |
| mova m2, [esp+0x80] |
| mova m4, [esp+0x70] |
| mova [esp+0x70], m5 |
| mova m5, [esp+0x60] |
| mova [esp+0x80], m6 |
| mova [esp+0x60], m7 |
| psubd m1, [aq-(384+16)*4*2] ; a:ctr+bottom [first half] |
| psubd m3, [aq-(384+16)*4*2+16] ; a:ctr+bottom [second half] |
| %endif |
| |
| add aq, (384+16)*4 |
| add bq, (384+16)*2 |
| add tq, 384*2 |
| add srcq, strideq |
| dec yd |
| jg .loop_y |
| add xd, 8 |
| cmp xd, wd |
| jl .loop_x |
| RET |
| |
| cglobal sgr_weighted1, 4, 7, 8, dst, stride, t, w, h, wt |
| movifnidn hd, hm |
| %if ARCH_X86_32 |
| SETUP_PIC r6, 0 |
| %endif |
| movd m0, wtm |
| pshufb m0, [PIC_sym(pb_0_1)] |
| psllw m0, 4 |
| pxor m7, m7 |
| DEFINE_ARGS dst, stride, t, w, h, idx |
| .loop_y: |
| xor idxd, idxd |
| .loop_x: |
| mova m1, [tq+idxq*2+ 0] |
| mova m4, [tq+idxq*2+16] |
| mova m5, [dstq+idxq] |
| punpcklbw m2, m5, m7 |
| punpckhbw m5, m7 |
| psllw m3, m2, 4 |
| psllw m6, m5, 4 |
| psubw m1, m3 |
| psubw m4, m6 |
| pmulhrsw m1, m0 |
| pmulhrsw m4, m0 |
| paddw m1, m2 |
| paddw m4, m5 |
| packuswb m1, m4 |
| mova [dstq+idxq], m1 |
| add idxd, 16 |
| cmp idxd, wd |
| jl .loop_x |
| add dstq, strideq |
| add tq, 384 * 2 |
| dec hd |
| jg .loop_y |
| RET |
| |
| %if ARCH_X86_64 |
| cglobal sgr_box5_h, 5, 11, 12, sumsq, sum, left, src, stride, w, h, edge, x, xlim |
| mov edged, edgem |
| movifnidn wd, wm |
| mov hd, hm |
| mova m10, [pb_0] |
| mova m11, [pb_0_1] |
| %else |
| cglobal sgr_box5_h, 7, 7, 8, sumsq, sum, left, src, xlim, x, h, edge |
| %define edged edgemp |
| %define wd xd |
| %define wq wd |
| %define wm r5m |
| %define strideq r4m |
| SUB esp, 8 |
| SETUP_PIC sumsqd, 1, 1 |
| |
| %define m10 [PIC_sym(pb_0)] |
| %define m11 [PIC_sym(pb_0_1)] |
| %endif |
| |
| test edged, 2 ; have_right |
| jz .no_right |
| xor xlimd, xlimd |
| add wd, 2 |
| add wd, 15 |
| and wd, ~15 |
| jmp .right_done |
| .no_right: |
| mov xlimd, 3 |
| dec wd |
| .right_done: |
| pxor m1, m1 |
| lea srcq, [srcq+wq+1] |
| lea sumq, [sumq+wq*2-2] |
| lea sumsqq, [sumsqq+wq*4-4] |
| neg wq |
| %if ARCH_X86_64 |
| lea r10, [pb_right_ext_mask+16] |
| %else |
| mov wm, xd |
| %define wq wm |
| %endif |
| |
| .loop_y: |
| mov xq, wq |
| ; load left |
| test edged, 1 ; have_left |
| jz .no_left |
| test leftq, leftq |
| jz .load_left_from_main |
| movd m0, [leftq] |
| movd m2, [srcq+xq-1] |
| pslldq m2, 4 |
| por m0, m2 |
| pslldq m0, 11 |
| add leftq, 4 |
| jmp .expand_x |
| .no_left: |
| movd m0, [srcq+xq-1] |
| XCHG_PIC_REG |
| pshufb m0, m10 |
| XCHG_PIC_REG |
| jmp .expand_x |
| .load_left_from_main: |
| movd m0, [srcq+xq-4] |
| pslldq m0, 12 |
| .expand_x: |
| punpckhbw m0, m1 |
| |
| ; when we reach this, m0 contains left two px in highest words |
| cmp xq, -8 |
| jle .loop_x |
| test xq, xq |
| jge .right_extend |
| .partial_load_and_extend: |
| XCHG_PIC_REG |
| movd m3, [srcq-1] |
| movq m2, [srcq+xq] |
| pshufb m3, m10 |
| punpcklbw m3, m1 |
| punpcklbw m2, m1 |
| %if ARCH_X86_64 |
| movu m4, [r10+xq*2] |
| %else |
| movu m4, [PIC_sym(pb_right_ext_mask+16)+xd*2] |
| XCHG_PIC_REG |
| %endif |
| pand m2, m4 |
| pandn m4, m3 |
| por m2, m4 |
| jmp .loop_x_noload |
| .right_extend: |
| psrldq m2, m0, 14 |
| XCHG_PIC_REG |
| pshufb m2, m11 |
| XCHG_PIC_REG |
| jmp .loop_x_noload |
| |
| .loop_x: |
| movq m2, [srcq+xq] |
| punpcklbw m2, m1 |
| .loop_x_noload: |
| palignr m3, m2, m0, 8 |
| palignr m4, m2, m0, 10 |
| palignr m5, m2, m0, 12 |
| palignr m6, m2, m0, 14 |
| |
| %if ARCH_X86_64 |
| paddw m0, m3, m2 |
| punpcklwd m7, m3, m2 |
| punpckhwd m3, m2 |
| paddw m0, m4 |
| punpcklwd m8, m4, m5 |
| punpckhwd m4, m5 |
| paddw m0, m5 |
| punpcklwd m9, m6, m1 |
| punpckhwd m5, m6, m1 |
| paddw m0, m6 |
| pmaddwd m7, m7 |
| pmaddwd m3, m3 |
| pmaddwd m8, m8 |
| pmaddwd m4, m4 |
| pmaddwd m9, m9 |
| pmaddwd m5, m5 |
| paddd m7, m8 |
| paddd m3, m4 |
| paddd m7, m9 |
| paddd m3, m5 |
| movu [sumq+xq*2], m0 |
| movu [sumsqq+xq*4+ 0], m7 |
| movu [sumsqq+xq*4+16], m3 |
| %else |
| paddw m0, m3, m2 |
| paddw m0, m4 |
| paddw m0, m5 |
| paddw m0, m6 |
| movu [sumq+xq*2], m0 |
| punpcklwd m7, m3, m2 |
| punpckhwd m3, m2 |
| punpcklwd m0, m4, m5 |
| punpckhwd m4, m5 |
| punpckhwd m5, m6, m1 |
| pmaddwd m7, m7 |
| pmaddwd m3, m3 |
| pmaddwd m0, m0 |
| pmaddwd m4, m4 |
| pmaddwd m5, m5 |
| paddd m7, m0 |
| paddd m3, m4 |
| paddd m3, m5 |
| punpcklwd m0, m6, m1 |
| pmaddwd m0, m0 |
| paddd m7, m0 |
| movu [sumsqq+xq*4+ 0], m7 |
| movu [sumsqq+xq*4+16], m3 |
| %endif |
| |
| mova m0, m2 |
| add xq, 8 |
| |
| ; if x <= -8 we can reload more pixels |
| ; else if x < 0 we reload and extend (this implies have_right=0) |
| ; else if x < xlimd we extend from previous load (this implies have_right=0) |
| ; else we are done |
| |
| cmp xq, -8 |
| jle .loop_x |
| test xq, xq |
| jl .partial_load_and_extend |
| cmp xq, xlimq |
| jl .right_extend |
| |
| add sumsqq, (384+16)*4 |
| add sumq, (384+16)*2 |
| add srcq, strideq |
| dec hd |
| jg .loop_y |
| %if ARCH_X86_32 |
| ADD esp, 8 |
| %endif |
| RET |
| |
| %if ARCH_X86_64 |
| cglobal sgr_box5_v, 4, 10, 15, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim |
| movifnidn edged, edgem |
| mov ylimd, edged |
| %else |
| cglobal sgr_box5_v, 5, 7, 8, -44, sumsq, sum, x, y, ylim, sumsq_ptr, sum_ptr |
| %define wm [esp+0] |
| %define hm [esp+4] |
| %define edgem [esp+8] |
| mov wm, xd |
| mov hm, yd |
| mov edgem, ylimd |
| %endif |
| |
| and ylimd, 8 ; have_bottom |
| shr ylimd, 2 |
| sub ylimd, 3 ; -3 if have_bottom=0, else -1 |
| mov xq, -2 |
| %if ARCH_X86_64 |
| .loop_x: |
| lea yd, [hd+ylimd+2] |
| lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] |
| lea sum_ptrq, [ sumq+xq*2+2-(384+16)*2] |
| test edged, 4 ; have_top |
| jnz .load_top |
| movu m0, [sumsq_ptrq+(384+16)*4*1] |
| movu m1, [sumsq_ptrq+(384+16)*4*1+16] |
| mova m2, m0 |
| mova m3, m1 |
| mova m4, m0 |
| mova m5, m1 |
| mova m6, m0 |
| mova m7, m1 |
| movu m10, [sum_ptrq+(384+16)*2*1] |
| mova m11, m10 |
| mova m12, m10 |
| mova m13, m10 |
| jmp .loop_y_second_load |
| .load_top: |
| movu m0, [sumsq_ptrq-(384+16)*4*1] ; l3/4sq [left] |
| movu m1, [sumsq_ptrq-(384+16)*4*1+16] ; l3/4sq [right] |
| movu m4, [sumsq_ptrq-(384+16)*4*0] ; l2sq [left] |
| movu m5, [sumsq_ptrq-(384+16)*4*0+16] ; l2sq [right] |
| mova m2, m0 |
| mova m3, m1 |
| movu m10, [sum_ptrq-(384+16)*2*1] ; l3/4 |
| movu m12, [sum_ptrq-(384+16)*2*0] ; l2 |
| mova m11, m10 |
| .loop_y: |
| movu m6, [sumsq_ptrq+(384+16)*4*1] ; l1sq [left] |
| movu m7, [sumsq_ptrq+(384+16)*4*1+16] ; l1sq [right] |
| movu m13, [sum_ptrq+(384+16)*2*1] ; l1 |
| .loop_y_second_load: |
| test yd, yd |
| jle .emulate_second_load |
| movu m8, [sumsq_ptrq+(384+16)*4*2] ; l0sq [left] |
| movu m9, [sumsq_ptrq+(384+16)*4*2+16] ; l0sq [right] |
| movu m14, [sum_ptrq+(384+16)*2*2] ; l0 |
| .loop_y_noload: |
| paddd m0, m2 |
| paddd m1, m3 |
| paddw m10, m11 |
| paddd m0, m4 |
| paddd m1, m5 |
| paddw m10, m12 |
| paddd m0, m6 |
| paddd m1, m7 |
| paddw m10, m13 |
| paddd m0, m8 |
| paddd m1, m9 |
| paddw m10, m14 |
| movu [sumsq_ptrq+ 0], m0 |
| movu [sumsq_ptrq+16], m1 |
| movu [sum_ptrq], m10 |
| |
| ; shift position down by one |
| mova m0, m4 |
| mova m1, m5 |
| mova m2, m6 |
| mova m3, m7 |
| mova m4, m8 |
| mova m5, m9 |
| mova m10, m12 |
| mova m11, m13 |
| mova m12, m14 |
| add sumsq_ptrq, (384+16)*4*2 |
| add sum_ptrq, (384+16)*2*2 |
| sub yd, 2 |
| jge .loop_y |
| ; l1 = l0 |
| mova m6, m8 |
| mova m7, m9 |
| mova m13, m14 |
| cmp yd, ylimd |
| jg .loop_y_noload |
| add xd, 8 |
| cmp xd, wd |
| jl .loop_x |
| RET |
| .emulate_second_load: |
| mova m8, m6 |
| mova m9, m7 |
| mova m14, m13 |
| jmp .loop_y_noload |
| %else |
| .sumsq_loop_x: |
| lea yd, [ylimd+2] |
| add yd, hm |
| lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] |
| test dword edgem, 4 ; have_top |
| jnz .sumsq_load_top |
| movu m0, [sumsq_ptrq+(384+16)*4*1] |
| movu m1, [sumsq_ptrq+(384+16)*4*1+16] |
| mova m4, m0 |
| mova m5, m1 |
| mova m6, m0 |
| mova m7, m1 |
| mova [esp+0x1c], m0 |
| mova [esp+0x0c], m1 |
| jmp .sumsq_loop_y_second_load |
| .sumsq_load_top: |
| movu m0, [sumsq_ptrq-(384+16)*4*1] ; l3/4sq [left] |
| movu m1, [sumsq_ptrq-(384+16)*4*1+16] ; l3/4sq [right] |
| movu m4, [sumsq_ptrq-(384+16)*4*0] ; l2sq [left] |
| movu m5, [sumsq_ptrq-(384+16)*4*0+16] ; l2sq [right] |
| mova [esp+0x1c], m0 |
| mova [esp+0x0c], m1 |
| .sumsq_loop_y: |
| movu m6, [sumsq_ptrq+(384+16)*4*1] ; l1sq [left] |
| movu m7, [sumsq_ptrq+(384+16)*4*1+16] ; l1sq [right] |
| .sumsq_loop_y_second_load: |
| test yd, yd |
| jle .sumsq_emulate_second_load |
| movu m2, [sumsq_ptrq+(384+16)*4*2] ; l0sq [left] |
| movu m3, [sumsq_ptrq+(384+16)*4*2+16] ; l0sq [right] |
| .sumsq_loop_y_noload: |
| paddd m0, [esp+0x1c] |
| paddd m1, [esp+0x0c] |
| paddd m0, m4 |
| paddd m1, m5 |
| paddd m0, m6 |
| paddd m1, m7 |
| paddd m0, m2 |
| paddd m1, m3 |
| movu [sumsq_ptrq+ 0], m0 |
| movu [sumsq_ptrq+16], m1 |
| |
| ; shift position down by one |
| mova m0, m4 |
| mova m1, m5 |
| mova m4, m2 |
| mova m5, m3 |
| mova [esp+0x1c], m6 |
| mova [esp+0x0c], m7 |
| add sumsq_ptrq, (384+16)*4*2 |
| sub yd, 2 |
| jge .sumsq_loop_y |
| ; l1 = l0 |
| mova m6, m2 |
| mova m7, m3 |
| cmp yd, ylimd |
| jg .sumsq_loop_y_noload |
| add xd, 8 |
| cmp xd, wm |
| jl .sumsq_loop_x |
| |
| mov xd, -2 |
| .sum_loop_x: |
| lea yd, [ylimd+2] |
| add yd, hm |
| lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] |
| test dword edgem, 4 ; have_top |
| jnz .sum_load_top |
| movu m0, [sum_ptrq+(384+16)*2*1] |
| mova m1, m0 |
| mova m2, m0 |
| mova m3, m0 |
| jmp .sum_loop_y_second_load |
| .sum_load_top: |
| movu m0, [sum_ptrq-(384+16)*2*1] ; l3/4 |
| movu m2, [sum_ptrq-(384+16)*2*0] ; l2 |
| mova m1, m0 |
| .sum_loop_y: |
| movu m3, [sum_ptrq+(384+16)*2*1] ; l1 |
| .sum_loop_y_second_load: |
| test yd, yd |
| jle .sum_emulate_second_load |
| movu m4, [sum_ptrq+(384+16)*2*2] ; l0 |
| .sum_loop_y_noload: |
| paddw m0, m1 |
| paddw m0, m2 |
| paddw m0, m3 |
| paddw m0, m4 |
| movu [sum_ptrq], m0 |
| |
| ; shift position down by one |
| mova m0, m2 |
| mova m1, m3 |
| mova m2, m4 |
| add sum_ptrq, (384+16)*2*2 |
| sub yd, 2 |
| jge .sum_loop_y |
| ; l1 = l0 |
| mova m3, m4 |
| cmp yd, ylimd |
| jg .sum_loop_y_noload |
| add xd, 8 |
| cmp xd, wm |
| jl .sum_loop_x |
| RET |
| .sumsq_emulate_second_load: |
| mova m2, m6 |
| mova m3, m7 |
| jmp .sumsq_loop_y_noload |
| .sum_emulate_second_load: |
| mova m4, m3 |
| jmp .sum_loop_y_noload |
| %endif |
| |
| cglobal sgr_calc_ab2, 4, 7, 11, a, b, w, h, s |
| movifnidn sd, sm |
| sub aq, (384+16-1)*4 |
| sub bq, (384+16-1)*2 |
| add hd, 2 |
| %if ARCH_X86_64 |
| LEA r5, sgr_x_by_x-0xF03 |
| %else |
| SETUP_PIC r5, 0 |
| %endif |
| movd m6, sd |
| pshuflw m6, m6, q0000 |
| punpcklqdq m6, m6 |
| pxor m7, m7 |
| DEFINE_ARGS a, b, w, h, x |
| %if ARCH_X86_64 |
| mova m8, [pd_0xF0080029] |
| mova m9, [pw_256] |
| psrld m10, m9, 15 ; pd_512 |
| %else |
| %define m8 [PIC_sym(pd_0xF0080029)] |
| %define m9 [PIC_sym(pw_256)] |
| %define m10 [PIC_sym(pd_512)] |
| %endif |
| .loop_y: |
| mov xq, -2 |
| .loop_x: |
| movq m0, [bq+xq*2+0] |
| movq m1, [bq+xq*2+8] |
| punpcklwd m0, m7 |
| punpcklwd m1, m7 |
| movu m2, [aq+xq*4+ 0] |
| movu m3, [aq+xq*4+16] |
| pslld m4, m2, 3 ; aa * 8 |
| pslld m5, m3, 3 |
| paddd m2, m4 ; aa * 9 |
| paddd m3, m5 |
| paddd m4, m4 ; aa * 16 |
| paddd m5, m5 |
| paddd m2, m4 ; aa * 25 |
| paddd m3, m5 |
| pmaddwd m4, m0, m0 |
| pmaddwd m5, m1, m1 |
| psubd m2, m4 ; p = aa * 25 - bb * bb |
| psubd m3, m5 |
| MULLD m2, m6 |
| MULLD m3, m6 |
| paddusw m2, m8 |
| paddusw m3, m8 |
| psrld m2, 20 ; z |
| psrld m3, 20 |
| GATHERDD m4, m2 ; xx |
| GATHERDD m2, m3 |
| psrld m4, 24 |
| psrld m2, 24 |
| packssdw m3, m4, m2 |
| pmullw m4, m8 |
| pmullw m2, m8 |
| psubw m5, m9, m3 |
| pmaddwd m0, m4 |
| pmaddwd m1, m2 |
| paddd m0, m10 |
| paddd m1, m10 |
| psrld m0, 10 |
| psrld m1, 10 |
| movu [bq+xq*2], m5 |
| movu [aq+xq*4+ 0], m0 |
| movu [aq+xq*4+16], m1 |
| add xd, 8 |
| cmp xd, wd |
| jl .loop_x |
| add aq, (384+16)*4*2 |
| add bq, (384+16)*2*2 |
| sub hd, 2 |
| jg .loop_y |
| RET |
| |
| %if ARCH_X86_64 |
| cglobal sgr_finish_filter2, 5, 13, 14, t, src, stride, a, b, w, h, \ |
| tmp_base, src_base, a_base, b_base, x, y |
| movifnidn wd, wm |
| mov hd, hm |
| mov tmp_baseq, tq |
| mov src_baseq, srcq |
| mov a_baseq, aq |
| mov b_baseq, bq |
| mova m9, [pw_5_6] |
| mova m12, [pw_256] |
| psrlw m10, m12, 8 ; pw_1 |
| psrlw m11, m12, 1 ; pw_128 |
| pxor m13, m13 |
| %else |
| cglobal sgr_finish_filter2, 6, 7, 8, t, src, stride, a, b, x, y |
| %define tmp_baseq r0m |
| %define src_baseq r1m |
| %define a_baseq r3m |
| %define b_baseq r4m |
| %define wd r5m |
| %define hd r6m |
| |
| SUB esp, 8 |
| SETUP_PIC yd |
| |
| %define m8 m5 |
| %define m9 [PIC_sym(pw_5_6)] |
| %define m10 [PIC_sym(pw_1)] |
| %define m11 [PIC_sym(pw_128)] |
| %define m12 [PIC_sym(pw_256)] |
| %define m13 m0 |
| %endif |
| xor xd, xd |
| .loop_x: |
| mov tq, tmp_baseq |
| mov srcq, src_baseq |
| mov aq, a_baseq |
| mov bq, b_baseq |
| movu m0, [aq+xq*4-(384+16)*4-4] |
| mova m1, [aq+xq*4-(384+16)*4] |
| movu m2, [aq+xq*4-(384+16)*4+4] |
| movu m3, [aq+xq*4-(384+16)*4-4+16] |
| mova m4, [aq+xq*4-(384+16)*4+16] |
| movu m5, [aq+xq*4-(384+16)*4+4+16] |
| paddd m0, m2 |
| paddd m3, m5 |
| paddd m0, m1 |
| paddd m3, m4 |
| pslld m2, m0, 2 |
| pslld m5, m3, 2 |
| paddd m2, m0 |
| paddd m5, m3 |
| paddd m0, m2, m1 ; prev_odd_b [first half] |
| paddd m1, m5, m4 ; prev_odd_b [second half] |
| movu m3, [bq+xq*2-(384+16)*2-2] |
| mova m4, [bq+xq*2-(384+16)*2] |
| movu m5, [bq+xq*2-(384+16)*2+2] |
| paddw m3, m5 |
| punpcklwd m5, m3, m4 |
| punpckhwd m3, m4 |
| pmaddwd m5, m9 |
| pmaddwd m3, m9 |
| mova m2, m5 |
| packssdw m2, m3 ; prev_odd_a |
| lea tq, [tq+xq*2] |
| lea srcq, [srcq+xq*1] |
| lea aq, [aq+xq*4+(384+16)*4] |
| lea bq, [bq+xq*2+(384+16)*2] |
| %if ARCH_X86_32 |
| mov [esp], PIC_reg |
| %endif |
| mov yd, hd |
| XCHG_PIC_REG |
| .loop_y: |
| movu m3, [aq-4] |
| mova m4, [aq] |
| movu m5, [aq+4] |
| paddd m3, m5 |
| paddd m3, m4 |
| pslld m5, m3, 2 |
| paddd m5, m3 |
| paddd m5, m4 ; cur_odd_b [first half] |
| movu m3, [aq+16-4] |
| mova m6, [aq+16] |
| movu m7, [aq+16+4] |
| paddd m3, m7 |
| paddd m3, m6 |
| pslld m7, m3, 2 |
| paddd m7, m3 |
| paddd m4, m7, m6 ; cur_odd_b [second half] |
| movu m3, [bq-2] |
| mova m6, [bq] |
| movu m7, [bq+2] |
| paddw m3, m7 |
| punpcklwd m7, m3, m6 |
| punpckhwd m3, m6 |
| pmaddwd m7, m9 |
| pmaddwd m3, m9 |
| packssdw m6, m7, m3 ; cur_odd_a |
| |
| paddd m0, m5 ; cur_even_b [first half] |
| paddd m1, m4 ; cur_even_b [second half] |
| paddw m2, m6 ; cur_even_a |
| |
| movq m3, [srcq] |
| %if ARCH_X86_64 |
| punpcklbw m3, m13 |
| %else |
| mova [td], m5 |
| pxor m7, m7 |
| punpcklbw m3, m7 |
| %endif |
| punpcklwd m7, m3, m10 |
| punpckhwd m3, m10 |
| punpcklwd m8, m2, m12 |
| punpckhwd m2, m12 |
| pmaddwd m7, m8 |
| pmaddwd m3, m2 |
| paddd m7, m0 |
| paddd m3, m1 |
| psrad m7, 9 |
| psrad m3, 9 |
| |
| %if ARCH_X86_32 |
| pxor m13, m13 |
| %endif |
| movq m8, [srcq+strideq] |
| punpcklbw m8, m13 |
| punpcklwd m0, m8, m10 |
| punpckhwd m8, m10 |
| punpcklwd m1, m6, m11 |
| punpckhwd m2, m6, m11 |
| pmaddwd m0, m1 |
| pmaddwd m8, m2 |
| %if ARCH_X86_64 |
| paddd m0, m5 |
| %else |
| paddd m0, [td] |
| %endif |
| paddd m8, m4 |
| psrad m0, 8 |
| psrad m8, 8 |
| |
| packssdw m7, m3 |
| packssdw m0, m8 |
| %if ARCH_X86_32 |
| mova m5, [td] |
| %endif |
| mova [tq+384*2*0], m7 |
| mova [tq+384*2*1], m0 |
| |
| mova m0, m5 |
| mova m1, m4 |
| mova m2, m6 |
| add aq, (384+16)*4*2 |
| add bq, (384+16)*2*2 |
| add tq, 384*2*2 |
| lea srcq, [srcq+strideq*2] |
| %if ARCH_X86_64 |
| sub yd, 2 |
| %else |
| sub dword [esp+4], 2 |
| %endif |
| jg .loop_y |
| add xd, 8 |
| cmp xd, wd |
| jl .loop_x |
| %if ARCH_X86_32 |
| ADD esp, 8 |
| %endif |
| RET |
| |
| cglobal sgr_weighted2, 4, 7, 12, dst, stride, t1, t2, w, h, wt |
| movifnidn wd, wm |
| mov wtq, wtmp |
| %if ARCH_X86_64 |
| movifnidn hd, hm |
| mova m10, [pd_1024] |
| pxor m11, m11 |
| %else |
| SETUP_PIC hd, 0 |
| %define m10 [PIC_sym(pd_1024)] |
| %define m11 m7 |
| %endif |
| movd m0, [wtq] |
| pshufd m0, m0, 0 |
| DEFINE_ARGS dst, stride, t1, t2, w, h, idx |
| %if ARCH_X86_32 |
| %define hd hmp |
| %endif |
| |
| .loop_y: |
| xor idxd, idxd |
| .loop_x: |
| mova m1, [t1q+idxq*2+ 0] |
| mova m2, [t1q+idxq*2+16] |
| mova m3, [t2q+idxq*2+ 0] |
| mova m4, [t2q+idxq*2+16] |
| mova m6, [dstq+idxq] |
| %if ARCH_X86_32 |
| pxor m11, m11 |
| %endif |
| punpcklbw m5, m6, m11 |
| punpckhbw m6, m11 |
| psllw m7, m5, 4 |
| psubw m1, m7 |
| psubw m3, m7 |
| psllw m7, m6, 4 |
| psubw m2, m7 |
| psubw m4, m7 |
| punpcklwd m7, m1, m3 |
| punpckhwd m1, m3 |
| punpcklwd m3, m2, m4 |
| punpckhwd m2, m4 |
| pmaddwd m7, m0 |
| pmaddwd m1, m0 |
| pmaddwd m3, m0 |
| pmaddwd m2, m0 |
| paddd m7, m10 |
| paddd m1, m10 |
| paddd m3, m10 |
| paddd m2, m10 |
| psrad m7, 11 |
| psrad m1, 11 |
| psrad m3, 11 |
| psrad m2, 11 |
| packssdw m7, m1 |
| packssdw m3, m2 |
| paddw m7, m5 |
| paddw m3, m6 |
| packuswb m7, m3 |
| mova [dstq+idxq], m7 |
| add idxd, 16 |
| cmp idxd, wd |
| jl .loop_x |
| add dstq, strideq |
| add t1q, 384 * 2 |
| add t2q, 384 * 2 |
| dec hd |
| jg .loop_y |
| RET |
