| ; Copyright © 2019, VideoLAN and dav1d authors |
| ; Copyright © 2019, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "ext/x86/x86inc.asm" |
| |
| %if ARCH_X86_64 |
| |
| SECTION_RODATA 32 |
| pw_1024: times 16 dw 1024 |
| pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 |
| rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 |
| byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 |
| pw_seed_xor: times 2 dw 0xb524 |
| times 2 dw 0x49d8 |
| pd_m65536: dd ~0xffff |
| pb_23_22: times 2 db 23, 22 |
| pb_1: times 4 db 1 |
| hmul_bits: dw 32768, 16384, 8192, 4096 |
| round: dw 2048, 1024, 512 |
| mul_bits: dw 256, 128, 64, 32, 16 |
| round_vals: dw 32, 64, 128, 256, 512 |
| max: dw 255, 240, 235 |
| min: dw 0, 16 |
| pb_27_17_17_27: db 27, 17, 17, 27 |
| |
| %macro JMP_TABLE 1-* |
| %xdefine %1_table %%table |
| %xdefine %%base %1_table |
| %xdefine %%prefix mangle(private_prefix %+ _%1) |
| %%table: |
| %rep %0 - 1 |
| dd %%prefix %+ .ar%2 - %%base |
| %rotate 1 |
| %endrep |
| %endmacro |
| |
| JMP_TABLE generate_grain_y_avx2, 0, 1, 2, 3 |
| JMP_TABLE generate_grain_uv_420_avx2, 0, 1, 2, 3 |
| |
| struc FGData |
| .seed: resd 1 |
| .num_y_points: resd 1 |
| .y_points: resb 14 * 2 |
| .chroma_scaling_from_luma: resd 1 |
| .num_uv_points: resd 2 |
| .uv_points: resb 2 * 10 * 2 |
| .scaling_shift: resd 1 |
| .ar_coeff_lag: resd 1 |
| .ar_coeffs_y: resb 24 |
| .ar_coeffs_uv: resb 2 * 26 ; includes padding |
| .ar_coeff_shift: resd 1 |
| .grain_scale_shift: resd 1 |
| .uv_mult: resd 2 |
| .uv_luma_mult: resd 2 |
| .uv_offset: resd 2 |
| .overlap_flag: resd 1 |
| .clip_to_restricted_range: resd 1 |
| endstruc |
| |
| cextern gaussian_sequence |
| |
| SECTION .text |
| |
| INIT_XMM avx2 |
| cglobal generate_grain_y, 2, 9, 16, buf, fg_data |
| lea r4, [pb_mask] |
| %define base r4-pb_mask |
| movq xm1, [base+rnd_next_upperbit_mask] |
| movq xm4, [base+mul_bits] |
| movq xm7, [base+hmul_bits] |
| mov r2d, [fg_dataq+FGData.grain_scale_shift] |
| vpbroadcastw xm8, [base+round+r2*2] |
| mova xm5, [base+pb_mask] |
| vpbroadcastw xm0, [fg_dataq+FGData.seed] |
| vpbroadcastd xm9, [base+pd_m65536] |
| mov r2, -73*82 |
| sub bufq, r2 |
| lea r3, [gaussian_sequence] |
| .loop: |
| pand xm2, xm0, xm1 |
| psrlw xm3, xm2, 10 |
| por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set |
| pmullw xm2, xm4 ; bits 0x0f00 are set |
| pshufb xm2, xm5, xm2 ; set 15th bit for next 4 seeds |
| psllq xm6, xm2, 30 |
| por xm2, xm6 |
| psllq xm6, xm2, 15 |
| por xm2, xm6 ; aggregate each bit into next seed's high bit |
| pmulhuw xm3, xm0, xm7 |
| por xm2, xm3 ; 4 next output seeds |
| pshuflw xm0, xm2, q3333 |
| psrlw xm2, 5 |
| pmovzxwd xm3, xm2 |
| mova xm6, xm9 |
| vpgatherdd xm2, [r3+xm3*2], xm6 |
| pandn xm2, xm9, xm2 |
| packusdw xm2, xm2 |
| pmulhrsw xm2, xm8 |
| packsswb xm2, xm2 |
| movd [bufq+r2], xm2 |
| add r2, 4 |
| jl .loop |
| |
| ; auto-regression code |
| movsxd r2, [fg_dataq+FGData.ar_coeff_lag] |
| movsxd r2, [base+generate_grain_y_avx2_table+r2*4] |
| lea r2, [r2+base+generate_grain_y_avx2_table] |
| jmp r2 |
| |
| .ar1: |
| DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] |
| movd xm4, [fg_dataq+FGData.ar_coeffs_y] |
| DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 |
| pinsrb xm4, [pb_1], 3 |
| pmovsxbw xm4, xm4 |
| pshufd xm5, xm4, q1111 |
| pshufd xm4, xm4, q0000 |
| vpbroadcastw xm3, [base+round_vals+shiftq*2-12] ; rnd |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| mov mind, -128 |
| mov maxd, 127 |
| .y_loop_ar1: |
| mov xq, -76 |
| movsx val3d, byte [bufq+xq-1] |
| .x_loop_ar1: |
| pmovsxbw xm0, [bufq+xq-82-1] ; top/left |
| pmovsxbw xm2, [bufq+xq-82+0] ; top |
| pmovsxbw xm1, [bufq+xq-82+1] ; top/right |
| punpcklwd xm0, xm2 |
| punpcklwd xm1, xm3 |
| pmaddwd xm0, xm4 |
| pmaddwd xm1, xm5 |
| paddd xm0, xm1 |
| .x_loop_ar1_inner: |
| movd val0d, xm0 |
| psrldq xm0, 4 |
| imul val3d, cf3d |
| add val3d, val0d |
| %if WIN64 |
| sarx val3d, val3d, shiftd |
| %else |
| sar val3d, shiftb |
| %endif |
| movsx val0d, byte [bufq+xq] |
| add val3d, val0d |
| cmp val3d, maxd |
| cmovg val3d, maxd |
| cmp val3d, mind |
| cmovl val3d, mind |
| mov byte [bufq+xq], val3b |
| ; keep val3d in-place as left for next x iteration |
| inc xq |
| jz .x_loop_ar1_end |
| test xq, 3 |
| jnz .x_loop_ar1_inner |
| jmp .x_loop_ar1 |
| |
| .x_loop_ar1_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar1 |
| .ar0: |
| RET |
| |
| .ar2: |
| DEFINE_ARGS buf, fg_data, shift |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movd xm14, [base+hmul_bits-10+shiftq*2] |
| movq xm15, [base+byte_blend+1] |
| pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 |
| movd xm9, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 |
| pmovsxbw xm9, xm9 |
| DEFINE_ARGS buf, h, x |
| pshufd xm12, xm9, q0000 |
| pshufd xm13, xm9, q1111 |
| pshufd xm11, xm8, q3333 |
| pshufd xm10, xm8, q2222 |
| pshufd xm9, xm8, q1111 |
| pshufd xm8, xm8, q0000 |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| .y_loop_ar2: |
| mov xq, -76 |
| |
| .x_loop_ar2: |
| pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] |
| pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] |
| psrldq xm2, xm0, 2 ; y=-2,x=[-1,+5] |
| psrldq xm3, xm1, 2 ; y=-1,x=[-1,+5] |
| psrldq xm4, xm1, 4 ; y=-1,x=[+0,+5] |
| punpcklwd xm2, xm0, xm2 |
| punpcklwd xm3, xm4 |
| pmaddwd xm2, xm8 |
| pmaddwd xm3, xm11 |
| paddd xm2, xm3 |
| |
| psrldq xm4, xm0, 4 ; y=-2,x=[+0,+5] |
| psrldq xm5, xm0, 6 ; y=-2,x=[+1,+5] |
| psrldq xm6, xm0, 8 ; y=-2,x=[+2,+5] |
| punpcklwd xm4, xm5 |
| punpcklwd xm6, xm1 |
| psrldq xm7, xm1, 6 ; y=-1,x=[+1,+5] |
| psrldq xm1, xm1, 8 ; y=-1,x=[+2,+5] |
| punpcklwd xm7, xm1 |
| pmaddwd xm4, xm9 |
| pmaddwd xm6, xm10 |
| pmaddwd xm7, xm12 |
| paddd xm4, xm6 |
| paddd xm2, xm7 |
| paddd xm2, xm4 |
| |
| movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] |
| .x_loop_ar2_inner: |
| pmovsxbw xm1, xm0 |
| pmaddwd xm3, xm1, xm13 |
| paddd xm3, xm2 |
| psrldq xm1, 4 ; y=0,x=0 |
| psrldq xm2, 4 ; shift top to next pixel |
| psrad xm3, 5 |
| packssdw xm3, xm3 |
| pmulhrsw xm3, xm14 |
| paddw xm3, xm1 |
| packsswb xm3, xm3 |
| pextrb [bufq+xq], xm3, 0 |
| pslldq xm3, 2 |
| pand xm3, xm15 |
| pandn xm0, xm15, xm0 |
| por xm0, xm3 |
| psrldq xm0, 1 |
| inc xq |
| jz .x_loop_ar2_end |
| test xq, 3 |
| jnz .x_loop_ar2_inner |
| jmp .x_loop_ar2 |
| |
| .x_loop_ar2_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar2 |
| RET |
| |
| .ar3: |
| DEFINE_ARGS buf, fg_data, shift |
| %if WIN64 |
| SUB rsp, 16*12 |
| %assign stack_size_padded (stack_size_padded+16*12) |
| %assign stack_size (stack_size+16*12) |
| %else |
| ALLOC_STACK 16*12 |
| %endif |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movd xm14, [base+hmul_bits-10+shiftq*2] |
| movq xm15, [base+byte_blend] |
| pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-7 |
| pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_y+ 8] ; cf8-15 |
| pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 |
| pshufd xm9, xm0, q1111 |
| pshufd xm10, xm0, q2222 |
| pshufd xm11, xm0, q3333 |
| pshufd xm0, xm0, q0000 |
| pshufd xm6, xm1, q1111 |
| pshufd xm7, xm1, q2222 |
| pshufd xm8, xm1, q3333 |
| pshufd xm1, xm1, q0000 |
| pshufd xm3, xm2, q1111 |
| pshufd xm4, xm2, q2222 |
| psrldq xm5, xm2, 10 |
| pshufd xm2, xm2, q0000 |
| pinsrw xm5, [base+round_vals+shiftq*2-10], 3 |
| mova [rsp+ 0*16], xm0 |
| mova [rsp+ 1*16], xm9 |
| mova [rsp+ 2*16], xm10 |
| mova [rsp+ 3*16], xm11 |
| mova [rsp+ 4*16], xm1 |
| mova [rsp+ 5*16], xm6 |
| mova [rsp+ 6*16], xm7 |
| mova [rsp+ 7*16], xm8 |
| mova [rsp+ 8*16], xm2 |
| mova [rsp+ 9*16], xm3 |
| mova [rsp+10*16], xm4 |
| mova [rsp+11*16], xm5 |
| pxor xm13, xm13 |
| DEFINE_ARGS buf, h, x |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| .y_loop_ar3: |
| mov xq, -76 |
| |
| .x_loop_ar3: |
| movu xm0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] |
| movu xm1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] |
| movu xm2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] |
| pxor xm3, xm3 |
| pcmpgtb xm6, xm3, xm2 |
| pcmpgtb xm5, xm3, xm1 |
| pcmpgtb xm4, xm3, xm0 |
| punpckhbw xm3, xm0, xm4 |
| punpcklbw xm0, xm4 |
| punpckhbw xm4, xm1, xm5 |
| punpcklbw xm1, xm5 |
| punpckhbw xm5, xm2, xm6 |
| punpcklbw xm2, xm6 |
| |
| psrldq xm6, xm0, 2 |
| psrldq xm7, xm0, 4 |
| psrldq xm8, xm0, 6 |
| psrldq xm9, xm0, 8 |
| palignr xm10, xm3, xm0, 10 |
| palignr xm11, xm3, xm0, 12 |
| |
| punpcklwd xm0, xm6 |
| punpcklwd xm7, xm8 |
| punpcklwd xm9, xm10 |
| punpcklwd xm11, xm1 |
| pmaddwd xm0, [rsp+ 0*16] |
| pmaddwd xm7, [rsp+ 1*16] |
| pmaddwd xm9, [rsp+ 2*16] |
| pmaddwd xm11, [rsp+ 3*16] |
| paddd xm0, xm7 |
| paddd xm9, xm11 |
| paddd xm0, xm9 |
| |
| psrldq xm6, xm1, 2 |
| psrldq xm7, xm1, 4 |
| psrldq xm8, xm1, 6 |
| psrldq xm9, xm1, 8 |
| palignr xm10, xm4, xm1, 10 |
| palignr xm11, xm4, xm1, 12 |
| psrldq xm12, xm2, 2 |
| |
| punpcklwd xm6, xm7 |
| punpcklwd xm8, xm9 |
| punpcklwd xm10, xm11 |
| punpcklwd xm12, xm2, xm12 |
| pmaddwd xm6, [rsp+ 4*16] |
| pmaddwd xm8, [rsp+ 5*16] |
| pmaddwd xm10, [rsp+ 6*16] |
| pmaddwd xm12, [rsp+ 7*16] |
| paddd xm6, xm8 |
| paddd xm10, xm12 |
| paddd xm6, xm10 |
| paddd xm0, xm6 |
| |
| psrldq xm6, xm2, 4 |
| psrldq xm7, xm2, 6 |
| psrldq xm8, xm2, 8 |
| palignr xm9, xm5, xm2, 10 |
| palignr xm5, xm5, xm2, 12 |
| |
| punpcklwd xm6, xm7 |
| punpcklwd xm8, xm9 |
| punpcklwd xm5, xm13 |
| pmaddwd xm6, [rsp+ 8*16] |
| pmaddwd xm8, [rsp+ 9*16] |
| pmaddwd xm5, [rsp+10*16] |
| paddd xm0, xm6 |
| paddd xm8, xm5 |
| paddd xm0, xm8 |
| |
| movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] |
| .x_loop_ar3_inner: |
| pmovsxbw xm2, xm1 |
| pmaddwd xm2, [rsp+16*11] |
| pshufd xm3, xm2, q1111 |
| paddd xm2, xm3 ; left+cur |
| paddd xm2, xm0 ; add top |
| psrldq xm0, 4 |
| psrad xm2, 5 |
| packssdw xm2, xm2 |
| pmulhrsw xm2, xm14 |
| packsswb xm2, xm2 |
| pextrb [bufq+xq], xm2, 0 |
| pslldq xm2, 3 |
| pand xm2, xm15 |
| pandn xm1, xm15, xm1 |
| por xm1, xm2 |
| psrldq xm1, 1 |
| inc xq |
| jz .x_loop_ar3_end |
| test xq, 3 |
| jnz .x_loop_ar3_inner |
| jmp .x_loop_ar3 |
| |
| .x_loop_ar3_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar3 |
| RET |
| |
| INIT_XMM avx2 |
| cglobal generate_grain_uv_420, 4, 10, 16, buf, bufy, fg_data, uv |
| lea r4, [pb_mask] |
| %define base r4-pb_mask |
| movq xm1, [base+rnd_next_upperbit_mask] |
| movq xm4, [base+mul_bits] |
| movq xm7, [base+hmul_bits] |
| mov r5d, [fg_dataq+FGData.grain_scale_shift] |
| vpbroadcastw xm8, [base+round+r5*2] |
| mova xm5, [base+pb_mask] |
| vpbroadcastw xm0, [fg_dataq+FGData.seed] |
| vpbroadcastw xm9, [base+pw_seed_xor+uvq*4] |
| pxor xm0, xm9 |
| vpbroadcastd xm9, [base+pd_m65536] |
| lea r6, [gaussian_sequence] |
| mov r7d, 38 |
| add bufq, 44 |
| .loop_y: |
| mov r5, -44 |
| .loop_x: |
| pand xm2, xm0, xm1 |
| psrlw xm3, xm2, 10 |
| por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set |
| pmullw xm2, xm4 ; bits 0x0f00 are set |
| pshufb xm2, xm5, xm2 ; set 15th bit for next 4 seeds |
| psllq xm6, xm2, 30 |
| por xm2, xm6 |
| psllq xm6, xm2, 15 |
| por xm2, xm6 ; aggregate each bit into next seed's high bit |
| pmulhuw xm3, xm0, xm7 |
| por xm2, xm3 ; 4 next output seeds |
| pshuflw xm0, xm2, q3333 |
| psrlw xm2, 5 |
| pmovzxwd xm3, xm2 |
| mova xm6, xm9 |
| vpgatherdd xm2, [r6+xm3*2], xm6 |
| pandn xm2, xm9, xm2 |
| packusdw xm2, xm2 |
| pmulhrsw xm2, xm8 |
| packsswb xm2, xm2 |
| movd [bufq+r5], xm2 |
| add r5, 4 |
| jl .loop_x |
| add bufq, 82 |
| dec r7d |
| jg .loop_y |
| |
| ; auto-regression code |
| movsxd r5, [fg_dataq+FGData.ar_coeff_lag] |
| movsxd r5, [base+generate_grain_uv_420_avx2_table+r5*4] |
| lea r5, [r5+base+generate_grain_uv_420_avx2_table] |
| jmp r5 |
| |
| .ar0: |
| INIT_YMM avx2 |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| imul uvd, 25 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] |
| movd xm3, [base+hmul_bits+shiftq*2] |
| DEFINE_ARGS buf, bufy, h |
| pmovsxbw xm4, xm4 |
| vpbroadcastd m7, [pb_1] |
| vpbroadcastw m6, [hmul_bits+4] |
| vpbroadcastw m4, xm4 |
| vpbroadcastw m3, xm3 |
| sub bufq, 82*38+82-(82*3+41) |
| add bufyq, 3+82*3 |
| mov hd, 35 |
| .y_loop_ar0: |
| ; first 32 pixels |
| movu xm8, [bufyq] |
| movu xm9, [bufyq+82] |
| movu xm10, [bufyq+16] |
| movu xm11, [bufyq+82+16] |
| vinserti128 m8, [bufyq+32], 1 |
| vinserti128 m9, [bufyq+82+32], 1 |
| vinserti128 m10, [bufyq+48], 1 |
| vinserti128 m11, [bufyq+82+48], 1 |
| pmaddubsw m8, m7, m8 |
| pmaddubsw m9, m7, m9 |
| pmaddubsw m10, m7, m10 |
| pmaddubsw m11, m7, m11 |
| paddw m8, m9 |
| paddw m10, m11 |
| pmulhrsw m8, m6 |
| pmulhrsw m10, m6 |
| pmullw m8, m4 |
| pmullw m10, m4 |
| pmulhrsw m8, m3 |
| pmulhrsw m10, m3 |
| packsswb m8, m10 |
| movu m0, [bufq] |
| punpckhbw m1, m0, m8 |
| punpcklbw m0, m8 |
| pmaddubsw m1, m7, m1 |
| pmaddubsw m0, m7, m0 |
| packsswb m0, m1 |
| movu [bufq], m0 |
| |
| ; last 6 pixels |
| movu xm8, [bufyq+32*2] |
| movu xm9, [bufyq+32*2+82] |
| pmaddubsw xm8, xm7, xm8 |
| pmaddubsw xm9, xm7, xm9 |
| paddw xm8, xm9 |
| pmulhrsw xm8, xm6 |
| pmullw xm8, xm4 |
| pmulhrsw xm8, xm3 |
| packsswb xm8, xm8 |
| movq xm0, [bufq+32] |
| punpcklbw xm8, xm0 |
| pmaddubsw xm8, xm7, xm8 |
| packsswb xm8, xm8 |
| vpblendw xm0, xm8, xm0, 1000b |
| movq [bufq+32], xm0 |
| |
| add bufq, 82 |
| add bufyq, 82*2 |
| dec hd |
| jg .y_loop_ar0 |
| RET |
| |
| .ar1: |
| INIT_XMM avx2 |
| DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift |
| imul uvd, 25 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] |
| movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] |
| pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 |
| DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift |
| pmovsxbw xm4, xm4 |
| pshufd xm5, xm4, q1111 |
| pshufd xm4, xm4, q0000 |
| pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd |
| vpbroadcastd xm7, [pb_1] |
| vpbroadcastw xm6, [hmul_bits+4] |
| vpbroadcastd xm3, xm3 |
| sub bufq, 82*38+44-(82*3+41) |
| add bufyq, 79+82*3 |
| mov hd, 35 |
| mov mind, -128 |
| mov maxd, 127 |
| .y_loop_ar1: |
| mov xq, -38 |
| movsx val3d, byte [bufq+xq-1] |
| .x_loop_ar1: |
| pmovsxbw xm0, [bufq+xq-82-1] ; top/left |
| movq xm8, [bufyq+xq*2] |
| movq xm9, [bufyq+xq*2+82] |
| psrldq xm2, xm0, 2 ; top |
| psrldq xm1, xm0, 4 ; top/right |
| pmaddubsw xm8, xm7, xm8 |
| pmaddubsw xm9, xm7, xm9 |
| paddw xm8, xm9 |
| pmulhrsw xm8, xm6 |
| punpcklwd xm0, xm2 |
| punpcklwd xm1, xm8 |
| pmaddwd xm0, xm4 |
| pmaddwd xm1, xm5 |
| paddd xm0, xm1 |
| paddd xm0, xm3 |
| .x_loop_ar1_inner: |
| movd val0d, xm0 |
| psrldq xm0, 4 |
| imul val3d, cf3d |
| add val3d, val0d |
| sarx val3d, val3d, shiftd |
| movsx val0d, byte [bufq+xq] |
| add val3d, val0d |
| cmp val3d, maxd |
| cmovg val3d, maxd |
| cmp val3d, mind |
| cmovl val3d, mind |
| mov byte [bufq+xq], val3b |
| ; keep val3d in-place as left for next x iteration |
| inc xq |
| jz .x_loop_ar1_end |
| test xq, 3 |
| jnz .x_loop_ar1_inner |
| jmp .x_loop_ar1 |
| |
| .x_loop_ar1_end: |
| add bufq, 82 |
| add bufyq, 82*2 |
| dec hd |
| jg .y_loop_ar1 |
| RET |
| |
| .ar2: |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| imul uvd, 25 |
| movd xm15, [base+hmul_bits-10+shiftq*2] |
| pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 |
| pmovsxbw xm9, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 |
| vpbroadcastw xm7, [base+hmul_bits+4] |
| vpbroadcastd xm6, [base+pb_1] |
| DEFINE_ARGS buf, bufy, h, x |
| pshufd xm12, xm9, q0000 |
| pshufd xm13, xm9, q1111 |
| pshufd xm14, xm9, q2222 |
| pxor xm10, xm10 |
| vpblendw xm14, xm10, 10101010b |
| pshufd xm11, xm8, q3333 |
| pshufd xm10, xm8, q2222 |
| pshufd xm9, xm8, q1111 |
| pshufd xm8, xm8, q0000 |
| sub bufq, 82*38+44-(82*3+41) |
| add bufyq, 79+82*3 |
| mov hd, 35 |
| .y_loop_ar2: |
| mov xq, -38 |
| |
| .x_loop_ar2: |
| pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] |
| pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] |
| psrldq xm2, xm0, 2 ; y=-2,x=[-1,+5] |
| psrldq xm3, xm1, 2 ; y=-1,x=[-1,+5] |
| psrldq xm4, xm1, 4 ; y=-1,x=[+0,+5] |
| punpcklwd xm2, xm0, xm2 |
| punpcklwd xm3, xm4 |
| pmaddwd xm2, xm8 |
| pmaddwd xm3, xm11 |
| paddd xm2, xm3 |
| |
| psrldq xm4, xm0, 4 ; y=-2,x=[+0,+5] |
| psrldq xm5, xm0, 6 ; y=-2,x=[+1,+5] |
| psrldq xm0, 8 ; y=-2,x=[+2,+5] |
| punpcklwd xm4, xm5 |
| punpcklwd xm0, xm1 |
| psrldq xm3, xm1, 6 ; y=-1,x=[+1,+5] |
| psrldq xm1, xm1, 8 ; y=-1,x=[+2,+5] |
| punpcklwd xm3, xm1 |
| pmaddwd xm4, xm9 |
| pmaddwd xm0, xm10 |
| pmaddwd xm3, xm12 |
| paddd xm4, xm0 |
| paddd xm2, xm3 |
| paddd xm2, xm4 |
| |
| movq xm0, [bufyq+xq*2] |
| movq xm3, [bufyq+xq*2+82] |
| pmaddubsw xm0, xm6, xm0 |
| pmaddubsw xm3, xm6, xm3 |
| paddw xm0, xm3 |
| pmulhrsw xm0, xm7 |
| punpcklwd xm0, xm0 |
| pmaddwd xm0, xm14 |
| paddd xm2, xm0 |
| |
| movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] |
| .x_loop_ar2_inner: |
| pmovsxbw xm0, xm0 |
| pmaddwd xm3, xm0, xm13 |
| paddd xm3, xm2 |
| psrldq xm2, 4 ; shift top to next pixel |
| psrad xm3, 5 |
| packssdw xm3, xm3 |
| pmulhrsw xm3, xm15 |
| pslldq xm3, 2 |
| psrldq xm0, 2 |
| paddw xm3, xm0 |
| vpblendw xm0, xm3, 00000010b |
| packsswb xm0, xm0 |
| pextrb [bufq+xq], xm0, 1 |
| inc xq |
| jz .x_loop_ar2_end |
| test xq, 3 |
| jnz .x_loop_ar2_inner |
| jmp .x_loop_ar2 |
| |
| .x_loop_ar2_end: |
| add bufq, 82 |
| add bufyq, 82*2 |
| dec hd |
| jg .y_loop_ar2 |
| RET |
| |
| .ar3: |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| SUB rsp, 16*12 |
| %assign stack_size_padded (stack_size_padded+16*12) |
| %assign stack_size (stack_size+16*12) |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| imul uvd, 25 |
| movd xm14, [base+hmul_bits-10+shiftq*2] |
| pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-7 |
| pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 8] ; cf8-15 |
| pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-23 |
| pmovsxbw xm5, [fg_dataq+FGData.ar_coeffs_uv+uvq+24] ; cf24 [luma] |
| pshufd xm9, xm0, q1111 |
| pshufd xm10, xm0, q2222 |
| pshufd xm11, xm0, q3333 |
| pshufd xm0, xm0, q0000 |
| pshufd xm6, xm1, q1111 |
| pshufd xm7, xm1, q2222 |
| pshufd xm8, xm1, q3333 |
| pshufd xm1, xm1, q0000 |
| pshufd xm3, xm2, q1111 |
| pshufd xm4, xm2, q2222 |
| vpbroadcastw xm5, xm5 |
| vpblendw xm4, xm5, 10101010b ; interleave luma cf |
| psrldq xm5, xm2, 10 |
| pshufd xm2, xm2, q0000 |
| pinsrw xm5, [base+round_vals+shiftq*2-10], 3 |
| mova [rsp+ 0*16], xm0 |
| mova [rsp+ 1*16], xm9 |
| mova [rsp+ 2*16], xm10 |
| mova [rsp+ 3*16], xm11 |
| mova [rsp+ 4*16], xm1 |
| mova [rsp+ 5*16], xm6 |
| mova [rsp+ 6*16], xm7 |
| mova [rsp+ 7*16], xm8 |
| mova [rsp+ 8*16], xm2 |
| mova [rsp+ 9*16], xm3 |
| mova [rsp+10*16], xm4 |
| mova [rsp+11*16], xm5 |
| vpbroadcastd xm13, [base+pb_1] |
| vpbroadcastw xm15, [base+hmul_bits+4] |
| DEFINE_ARGS buf, bufy, h, x |
| sub bufq, 82*38+44-(82*3+41) |
| add bufyq, 79+82*3 |
| mov hd, 35 |
| .y_loop_ar3: |
| mov xq, -38 |
| |
| .x_loop_ar3: |
| movu xm0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] |
| movu xm1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] |
| movu xm2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] |
| pxor xm3, xm3 |
| pcmpgtb xm6, xm3, xm2 |
| pcmpgtb xm5, xm3, xm1 |
| pcmpgtb xm4, xm3, xm0 |
| punpckhbw xm3, xm0, xm4 |
| punpcklbw xm0, xm4 |
| punpckhbw xm4, xm1, xm5 |
| punpcklbw xm1, xm5 |
| punpckhbw xm5, xm2, xm6 |
| punpcklbw xm2, xm6 |
| |
| psrldq xm6, xm0, 2 |
| psrldq xm7, xm0, 4 |
| psrldq xm8, xm0, 6 |
| psrldq xm9, xm0, 8 |
| palignr xm10, xm3, xm0, 10 |
| palignr xm11, xm3, xm0, 12 |
| |
| punpcklwd xm0, xm6 |
| punpcklwd xm7, xm8 |
| punpcklwd xm9, xm10 |
| punpcklwd xm11, xm1 |
| pmaddwd xm0, [rsp+ 0*16] |
| pmaddwd xm7, [rsp+ 1*16] |
| pmaddwd xm9, [rsp+ 2*16] |
| pmaddwd xm11, [rsp+ 3*16] |
| paddd xm0, xm7 |
| paddd xm9, xm11 |
| paddd xm0, xm9 |
| |
| psrldq xm6, xm1, 2 |
| psrldq xm7, xm1, 4 |
| psrldq xm8, xm1, 6 |
| psrldq xm9, xm1, 8 |
| palignr xm10, xm4, xm1, 10 |
| palignr xm11, xm4, xm1, 12 |
| psrldq xm12, xm2, 2 |
| |
| punpcklwd xm6, xm7 |
| punpcklwd xm8, xm9 |
| punpcklwd xm10, xm11 |
| punpcklwd xm12, xm2, xm12 |
| pmaddwd xm6, [rsp+ 4*16] |
| pmaddwd xm8, [rsp+ 5*16] |
| pmaddwd xm10, [rsp+ 6*16] |
| pmaddwd xm12, [rsp+ 7*16] |
| paddd xm6, xm8 |
| paddd xm10, xm12 |
| paddd xm6, xm10 |
| paddd xm0, xm6 |
| |
| psrldq xm6, xm2, 4 |
| psrldq xm7, xm2, 6 |
| psrldq xm8, xm2, 8 |
| palignr xm9, xm5, xm2, 10 |
| palignr xm5, xm5, xm2, 12 |
| |
| movq xm1, [bufyq+xq*2] |
| movq xm2, [bufyq+xq*2+82] |
| pmaddubsw xm1, xm13, xm1 |
| pmaddubsw xm2, xm13, xm2 |
| paddw xm1, xm2 |
| pmulhrsw xm1, xm15 |
| |
| punpcklwd xm6, xm7 |
| punpcklwd xm8, xm9 |
| punpcklwd xm5, xm1 |
| pmaddwd xm6, [rsp+ 8*16] |
| pmaddwd xm8, [rsp+ 9*16] |
| pmaddwd xm5, [rsp+10*16] |
| paddd xm0, xm6 |
| paddd xm8, xm5 |
| paddd xm0, xm8 |
| |
| movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] |
| .x_loop_ar3_inner: |
| pmovsxbw xm1, xm1 |
| pmaddwd xm2, xm1, [rsp+16*11] |
| pshufd xm3, xm2, q1111 |
| paddd xm2, xm3 ; left+cur |
| paddd xm2, xm0 ; add top |
| psrldq xm0, 4 |
| psrad xm2, 5 |
| packssdw xm2, xm2 |
| pmulhrsw xm2, xm14 |
| pslldq xm2, 6 |
| vpblendw xm1, xm2, 1000b |
| packsswb xm1, xm1 |
| pextrb [bufq+xq], xm1, 3 |
| psrldq xm1, 1 |
| inc xq |
| jz .x_loop_ar3_end |
| test xq, 3 |
| jnz .x_loop_ar3_inner |
| jmp .x_loop_ar3 |
| |
| .x_loop_ar3_end: |
| add bufq, 82 |
| add bufyq, 82*2 |
| dec hd |
| jg .y_loop_ar3 |
| RET |
| |
| INIT_YMM avx2 |
| cglobal fgy_32x32xn, 6, 13, 16, dst, src, stride, fg_data, w, scaling, grain_lut |
| pcmpeqw m10, m10 |
| psrld m10, 24 |
| mov r7d, [fg_dataq+FGData.scaling_shift] |
| lea r8, [pb_mask] |
| %define base r8-pb_mask |
| vpbroadcastw m11, [base+mul_bits+r7*2-14] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| vpbroadcastw m12, [base+max+r7*4] |
| vpbroadcastw m13, [base+min+r7*2] |
| |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap |
| |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| movifnidn sbyd, sbym |
| test sbyd, sbyd |
| setnz r7b |
| test r7b, overlapb |
| jnz .vertical_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rol seed, 8 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| unused1, unused2, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| |
| .loop_x: |
| mov r6d, seed |
| or seed, 0xEFF4 |
| shr r6d, 1 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, overlap |
| |
| mov offxd, seed |
| rorx offyd, seed, 8 |
| shr offxd, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offyq, [offyq+offxq*2+747] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, overlap |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| .loop_y: |
| ; src |
| mova m0, [srcq] |
| pxor m2, m2 |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| punpckhwd m5, m0, m2 |
| punpcklwd m4, m0, m2 |
| punpckhwd m7, m1, m2 |
| punpcklwd m6, m1, m2 ; m4-7: src as dword |
| |
| ; scaling[src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m3, [grain_lutq+offxyq] |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hd |
| jg .loop_y |
| |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| test overlapd, overlapd |
| jz .loop_x |
| |
| ; r8m = sbym |
| movd xm15, [pb_27_17_17_27] |
| cmp dword r8m, 0 |
| jne .loop_x_hv_overlap |
| |
| ; horizontal overlap (without vertical overlap) |
| movd xm14, [pw_1024] |
| .loop_x_h_overlap: |
| mov r6d, seed |
| or seed, 0xEFF4 |
| shr r6d, 1 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy |
| |
| lea left_offxyd, [offyd+32] ; previous column's offy*stride+offx |
| mov offxd, seed |
| rorx offyd, seed, 8 |
| shr offxd, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offyq, [offyq+offxq*2+747] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| .loop_y_h_overlap: |
| ; src |
| mova m0, [srcq] |
| pxor m2, m2 |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| punpckhwd m5, m0, m2 |
| punpcklwd m4, m0, m2 |
| punpckhwd m7, m1, m2 |
| punpcklwd m6, m1, m2 ; m4-7: src as dword |
| |
| ; scaling[src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m3, [grain_lutq+offxyq] |
| movd xm4, [grain_lutq+left_offxyq] |
| punpcklbw xm4, xm3 |
| pmaddubsw xm4, xm15, xm4 |
| pmulhrsw xm4, xm14 |
| packsswb xm4, xm4 |
| vpblendw xm4, xm3, 11111110b |
| vpblendd m3, m4, 00001111b |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hd |
| jg .loop_y_h_overlap |
| |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne .loop_x_hv_overlap |
| jmp .loop_x_h_overlap |
| |
| .end: |
| RET |
| |
| .vertical_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap |
| |
| movzx sbyd, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, sbyd, 173 * 0x00010001 |
| imul sbyd, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add sbyd, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and sbyd, 0xff00ff00 |
| xor seed, r7d |
| xor seed, sbyd ; (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| unused1, unused2, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| |
| vpbroadcastd m14, [pw_1024] |
| .loop_x_v_overlap: |
| vpbroadcastw m15, [pb_27_17_17_27] |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, overlap, top_offxy |
| |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyq, [offyq+offxq*2+0x10001*747+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, top_offxy |
| |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| .loop_y_v_overlap: |
| ; src |
| mova m0, [srcq] |
| pxor m2, m2 |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| punpckhwd m5, m0, m2 |
| punpcklwd m4, m0, m2 |
| punpckhwd m7, m1, m2 |
| punpcklwd m6, m1, m2 ; m4-7: src as dword |
| |
| ; scaling[src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m3, [grain_lutq+offxyq] |
| movu m4, [grain_lutq+top_offxyq] |
| punpckhbw m6, m4, m3 |
| punpcklbw m4, m3 |
| pmaddubsw m6, m15, m6 |
| pmaddubsw m4, m15, m4 |
| pmulhrsw m6, m14 |
| pmulhrsw m4, m14 |
| packsswb m3, m4, m6 |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq+srcq], m0 |
| |
| vpbroadcastw m15, [pb_27_17_17_27+2] ; swap weights for second v-overlap line |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hw |
| jz .end_y_v_overlap |
| ; 2 lines get vertical overlap, then fall back to non-overlap code for |
| ; remaining (up to) 30 lines |
| xor hd, 0x10000 |
| test hd, 0x10000 |
| jnz .loop_y_v_overlap |
| jmp .loop_y |
| |
| .end_y_v_overlap: |
| add wq, 32 |
| jge .end_hv |
| lea srcq, [src_bakq+wq] |
| |
| ; since fg_dataq.overlap is guaranteed to be set, we never jump |
| ; back to .loop_x_v_overlap, and instead always fall-through to |
| ; h+v overlap |
| |
| movd xm15, [pb_27_17_17_27] |
| .loop_x_hv_overlap: |
| vpbroadcastw m8, [pb_27_17_17_27] |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, top_offxy, topleft_offxy |
| |
| lea topleft_offxyq, [top_offxyq+32] |
| lea left_offxyq, [offyq+32] |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyq, [offyq+offxq*2+0x10001*747+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, top_offxy, topleft_offxy |
| |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| .loop_y_hv_overlap: |
| ; src |
| mova m0, [srcq] |
| pxor m2, m2 |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| punpckhwd m5, m0, m2 |
| punpcklwd m4, m0, m2 |
| punpckhwd m7, m1, m2 |
| punpcklwd m6, m1, m2 ; m4-7: src as dword |
| |
| ; scaling[src] |
| pcmpeqw m3, m3 |
| ; FIXME it would be nice to have another register here to do 2 vpgatherdd's in parallel |
| vpgatherdd m9, [scalingq+m4], m3 |
| pcmpeqw m3, m3 |
| vpgatherdd m4, [scalingq+m5], m3 |
| pcmpeqw m3, m3 |
| vpgatherdd m5, [scalingq+m6], m3 |
| pcmpeqw m3, m3 |
| vpgatherdd m6, [scalingq+m7], m3 |
| pand m9, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m9, m4 |
| packusdw m5, m6 |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m3, [grain_lutq+offxyq] |
| movu m6, [grain_lutq+top_offxyq] |
| movd xm4, [grain_lutq+left_offxyq] |
| movd xm7, [grain_lutq+topleft_offxyq] |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw xm4, xm3 |
| punpcklbw xm7, xm6 |
| pmaddubsw xm4, xm15, xm4 |
| pmaddubsw xm7, xm15, xm7 |
| pmulhrsw xm4, xm14 |
| pmulhrsw xm7, xm14 |
| packsswb xm4, xm4 |
| packsswb xm7, xm7 |
| vpblendw xm4, xm3, 11111110b |
| vpblendw xm7, xm6, 11111110b |
| vpblendd m3, m4, 00001111b |
| vpblendd m6, m7, 00001111b |
| ; followed by v interpolation (top | cur -> cur) |
| punpckhbw m7, m6, m3 |
| punpcklbw m6, m3 |
| pmaddubsw m7, m8, m7 |
| pmaddubsw m6, m8, m6 |
| pmulhrsw m7, m14 |
| pmulhrsw m6, m14 |
| packsswb m3, m6, m7 |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmullw m2, m9 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq+srcq], m0 |
| |
| vpbroadcastw m8, [pb_27_17_17_27+2] ; swap weights for second v-overlap line |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hw |
| jz .end_y_hv_overlap |
| ; 2 lines get vertical overlap, then fall back to non-overlap code for |
| ; remaining (up to) 30 lines |
| xor hd, 0x10000 |
| test hd, 0x10000 |
| jnz .loop_y_hv_overlap |
| jmp .loop_y_h_overlap |
| |
| .end_y_hv_overlap: |
| add wq, 32 |
| lea srcq, [src_bakq+wq] |
| jl .loop_x_hv_overlap |
| |
| .end_hv: |
| RET |
| |
| cglobal fguv_32x32xn_i420, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ |
| grain_lut, h, sby, luma, lstride, uv_pl, is_id |
| pcmpeqw m10, m10 |
| psrld m10, 24 |
| mov r7d, [fg_dataq+FGData.scaling_shift] |
| lea r8, [pb_mask] |
| %define base r8-pb_mask |
| vpbroadcastw m11, [base+mul_bits+r7*2-14] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| mov r9d, dword is_idm |
| vpbroadcastw m13, [base+min+r7*2] |
| shlx r7d, r7d, r9d |
| vpbroadcastw m12, [base+max+r7*2] |
| |
| cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 |
| jne .csfl |
| |
| %macro FGUV_32x32xN_LOOP 1 ; not-csfl |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap |
| |
| %if %1 |
| mov r7d, dword r11m |
| vpbroadcastb m0, [fg_dataq+FGData.uv_mult+r7*4] |
| vpbroadcastb m1, [fg_dataq+FGData.uv_luma_mult+r7*4] |
| punpcklbw m14, m1, m0 |
| vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r7*4] |
| %else |
| vpbroadcastd m14, [pw_1024] |
| vpbroadcastd m15, [pb_23_22] |
| %endif |
| |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| movifnidn sbyd, sbym |
| test sbyd, sbyd |
| setnz r7b |
| test r7b, overlapb |
| jnz %%vertical_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rol seed, 8 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| unused2, unused3, see, overlap, unused4, unused5, lstride |
| |
| mov lumaq, r9mp |
| lea r12, [srcq+wq] |
| lea r13, [dstq+wq] |
| lea r14, [lumaq+wq*2] |
| mov r11mp, r12 |
| mov r12mp, r13 |
| mov lstrideq, r10mp |
| neg wq |
| |
| %%loop_x: |
| mov r6d, seed |
| or seed, 0xEFF4 |
| shr r6d, 1 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, overlap, unused1, unused2, lstride |
| |
| mov offxd, seed |
| rorx offyd, seed, 8 |
| shr offxd, 12 |
| and offyd, 0xf |
| imul offyd, 82 |
| lea offyq, [offyq+offxq+498] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, unused1, unused2, lstride |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| %%loop_y: |
| ; src |
| mova xm4, [lumaq+lstrideq*0+ 0] |
| mova xm6, [lumaq+lstrideq*0+16] |
| mova xm0, [srcq] |
| vpbroadcastd m7, [pb_1] |
| vinserti128 m4, [lumaq+lstrideq*2 +0], 1 |
| vinserti128 m6, [lumaq+lstrideq*2+16], 1 |
| vinserti128 m0, [srcq+strideq], 1 |
| pxor m2, m2 |
| pmaddubsw m4, m7 |
| pmaddubsw m6, m7 |
| pavgw m4, m2 |
| pavgw m6, m2 |
| |
| %if %1 |
| packuswb m4, m6 ; luma |
| punpckhbw m6, m4, m0 |
| punpcklbw m4, m0 ; { luma, chroma } |
| pmaddubsw m6, m14 |
| pmaddubsw m4, m14 |
| psraw m6, 6 |
| psraw m4, 6 |
| paddw m6, m15 |
| paddw m4, m15 |
| packuswb m4, m6 ; pack+unpack = clip |
| punpckhbw m6, m4, m2 |
| punpcklbw m4, m2 |
| %endif |
| |
| punpckhwd m5, m4, m2 |
| punpcklwd m4, m2 |
| punpckhwd m7, m6, m2 |
| punpcklwd m6, m2 ; m4-7: luma_src as dword |
| |
| ; scaling[luma_src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; unpack chroma_source |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu xm3, [grain_lutq+offxyq+ 0] |
| vinserti128 m3, [grain_lutq+offxyq+82], 1 |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*4] |
| add grain_lutq, 82*2 |
| sub hb, 2 |
| jg %%loop_y |
| |
| add wq, 16 |
| jge %%end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*2] |
| add srcq, wq |
| add dstq, wq |
| test overlapd, overlapd |
| jz %%loop_x |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne %%loop_x_hv_overlap |
| |
| ; horizontal overlap (without vertical overlap) |
| %%loop_x_h_overlap: |
| mov r6d, seed |
| or seed, 0xEFF4 |
| shr r6d, 1 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, unused1, unused2, lstride |
| |
| lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx |
| mov offxd, seed |
| rorx offyd, seed, 8 |
| shr offxd, 12 |
| and offyd, 0xf |
| imul offyd, 82 |
| lea offyq, [offyq+offxq+498] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, unused1, unused2, lstride |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| %%loop_y_h_overlap: |
| ; src |
| mova xm4, [lumaq+lstrideq*0+ 0] |
| mova xm6, [lumaq+lstrideq*0+16] |
| mova xm0, [srcq] |
| vpbroadcastd m7, [pb_1] |
| vinserti128 m4, [lumaq+lstrideq*2 +0], 1 |
| vinserti128 m6, [lumaq+lstrideq*2+16], 1 |
| vinserti128 m0, [srcq+strideq], 1 |
| pxor m2, m2 |
| pmaddubsw m4, m7 |
| pmaddubsw m6, m7 |
| pavgw m4, m2 |
| pavgw m6, m2 |
| |
| %if %1 |
| packuswb m4, m6 ; luma |
| punpckhbw m6, m4, m0 |
| punpcklbw m4, m0 ; { luma, chroma } |
| pmaddubsw m6, m14 |
| pmaddubsw m4, m14 |
| psraw m6, 6 |
| psraw m4, 6 |
| paddw m6, m15 |
| paddw m4, m15 |
| packuswb m4, m6 ; pack+unpack = clip |
| punpckhbw m6, m4, m2 |
| punpcklbw m4, m2 |
| %endif |
| |
| punpckhwd m5, m4, m2 |
| punpcklwd m4, m2 |
| punpckhwd m7, m6, m2 |
| punpcklwd m6, m2 ; m4-7: luma_src as dword |
| |
| ; scaling[luma_src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; unpack chroma_source |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %1 |
| vpbroadcastd m6, [pb_23_22] ; FIXME |
| %endif |
| movu xm3, [grain_lutq+offxyq+ 0] |
| movd xm4, [grain_lutq+left_offxyq+ 0] |
| vinserti128 m3, [grain_lutq+offxyq+82], 1 |
| vinserti128 m4, [grain_lutq+left_offxyq+82], 1 |
| punpcklbw m4, m3 |
| %if %1 |
| pmaddubsw m4, m6, m4 |
| pmulhrsw m4, [pw_1024] |
| %else |
| pmaddubsw m4, m15, m4 |
| pmulhrsw m4, m14 |
| %endif |
| packsswb m4, m4 |
| pcmpeqw m6, m6 ; FIXME |
| psrldq m6, 15 ; FIXME |
| vpblendvb m3, m3, m4, m6 |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*4] |
| add grain_lutq, 82*2 |
| sub hb, 2 |
| jg %%loop_y_h_overlap |
| |
| add wq, 16 |
| jge %%end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*2] |
| add srcq, wq |
| add dstq, wq |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne %%loop_x_hv_overlap |
| jmp %%loop_x_h_overlap |
| |
| %%end: |
| RET |
| |
| %%vertical_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ |
| sby, see, overlap, unused1, unused2, lstride |
| |
| movzx sbyd, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, sbyd, 173 * 0x00010001 |
| imul sbyd, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add sbyd, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and sbyd, 0xff00ff00 |
| xor seed, r7d |
| xor seed, sbyd ; (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| unused1, unused2, see, overlap, unused3, unused4, lstride |
| |
| mov lumaq, r9mp |
| lea r12, [srcq+wq] |
| lea r13, [dstq+wq] |
| lea r14, [lumaq+wq*2] |
| mov r11mp, r12 |
| mov r12mp, r13 |
| mov lstrideq, r10mp |
| neg wq |
| |
| %%loop_x_v_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, overlap, top_offxy, unused, lstride |
| |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 82 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyq, [offyq+offxq+0x10001*498+16*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, top_offxy, unused, lstride |
| |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| %%loop_y_v_overlap: |
| ; src |
| mova xm4, [lumaq+lstrideq*0+ 0] |
| mova xm6, [lumaq+lstrideq*0+16] |
| mova xm0, [srcq] |
| vpbroadcastd m7, [pb_1] |
| vinserti128 m4, [lumaq+lstrideq*2 +0], 1 |
| vinserti128 m6, [lumaq+lstrideq*2+16], 1 |
| vinserti128 m0, [srcq+strideq], 1 |
| pxor m2, m2 |
| pmaddubsw m4, m7 |
| pmaddubsw m6, m7 |
| pavgw m4, m2 |
| pavgw m6, m2 |
| |
| %if %1 |
| packuswb m4, m6 ; luma |
| punpckhbw m6, m4, m0 |
| punpcklbw m4, m0 ; { luma, chroma } |
| pmaddubsw m6, m14 |
| pmaddubsw m4, m14 |
| psraw m6, 6 |
| psraw m4, 6 |
| paddw m6, m15 |
| paddw m4, m15 |
| packuswb m4, m6 ; pack+unpack = clip |
| punpckhbw m6, m4, m2 |
| punpcklbw m4, m2 |
| %endif |
| |
| punpckhwd m5, m4, m2 |
| punpcklwd m4, m2 |
| punpckhwd m7, m6, m2 |
| punpcklwd m6, m2 ; m4-7: luma_src as dword |
| |
| ; scaling[luma_src] |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m8, [scalingq+m4], m3 |
| vpgatherdd m4, [scalingq+m5], m9 |
| pcmpeqw m3, m3 |
| pcmpeqw m9, m9 |
| vpgatherdd m5, [scalingq+m6], m3 |
| vpgatherdd m6, [scalingq+m7], m9 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; unpack chroma_source |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %1 |
| vpbroadcastd m6, [pb_23_22] |
| %endif |
| movq xm3, [grain_lutq+offxyq] |
| movq xm4, [grain_lutq+top_offxyq] |
| vinserti128 m3, [grain_lutq+offxyq+8], 1 |
| vinserti128 m4, [grain_lutq+top_offxyq+8], 1 |
| punpcklbw m4, m3 |
| %if %1 |
| pmaddubsw m4, m6, m4 |
| pmulhrsw m4, [pw_1024] |
| %else |
| pmaddubsw m4, m15, m4 |
| pmulhrsw m4, m14 |
| %endif |
| packsswb m4, m4 |
| vpermq m4, m4, q3120 |
| ; only interpolate first line, insert second line unmodified |
| vinserti128 m3, m4, [grain_lutq+offxyq+82], 1 |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| |
| sub hb, 2 |
| jl %%end_y_v_overlap |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*4] |
| add grain_lutq, 82*2 |
| jmp %%loop_y |
| |
| %%end_y_v_overlap: |
| add wq, 16 |
| jge %%end_hv |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*2] |
| add srcq, wq |
| add dstq, wq |
| |
| ; since fg_dataq.overlap is guaranteed to be set, we never jump |
| ; back to .loop_x_v_overlap, and instead always fall-through to |
| ; h+v overlap |
| |
| %%loop_x_hv_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride |
| |
| lea topleft_offxyq, [top_offxyq+16] |
| lea left_offxyq, [offyq+16] |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 82 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyq, [offyq+offxq+0x10001*498+16*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride |
| |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| %%loop_y_hv_overlap: |
| ; src |
| mova xm4, [lumaq+lstrideq*0+ 0] |
| mova xm6, [lumaq+lstrideq*0+16] |
| mova xm0, [srcq] |
| vpbroadcastd m7, [pb_1] |
| vinserti128 m4, [lumaq+lstrideq*2 +0], 1 |
| vinserti128 m6, [lumaq+lstrideq*2+16], 1 |
| vinserti128 m0, [srcq+strideq], 1 |
| pxor m2, m2 |
| pmaddubsw m4, m7 |
| pmaddubsw m6, m7 |
| pavgw m4, m2 |
| pavgw m6, m2 |
| |
| %if %1 |
| packuswb m4, m6 ; luma |
| punpckhbw m6, m4, m0 |
| punpcklbw m4, m0 ; { luma, chroma } |
| pmaddubsw m6, m14 |
| pmaddubsw m4, m14 |
| psraw m6, 6 |
| psraw m4, 6 |
| paddw m6, m15 |
| paddw m4, m15 |
| packuswb m4, m6 ; pack+unpack = clip |
| punpckhbw m6, m4, m2 |
| punpcklbw m4, m2 |
| %endif |
| |
| punpckhwd m5, m4, m2 |
| punpcklwd m4, m2 |
| punpckhwd m7, m6, m2 |
| punpcklwd m6, m2 ; m4-7: src as dword |
| |
| ; scaling[src] |
| pcmpeqw m9, m9 |
| pcmpeqw m3, m3 |
| vpgatherdd m8, [scalingq+m4], m9 |
| vpgatherdd m4, [scalingq+m5], m3 |
| pcmpeqw m9, m9 |
| pcmpeqw m3, m3 |
| vpgatherdd m5, [scalingq+m6], m9 |
| vpgatherdd m6, [scalingq+m7], m3 |
| pand m8, m10 |
| pand m4, m10 |
| pand m5, m10 |
| pand m6, m10 |
| packusdw m8, m4 |
| packusdw m5, m6 |
| |
| ; unpack chroma source |
| punpckhbw m1, m0, m2 |
| punpcklbw m0, m2 ; m0-1: src as word |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %1 |
| vpbroadcastd m9, [pb_23_22] |
| %endif |
| movu xm3, [grain_lutq+offxyq] |
| movq xm6, [grain_lutq+top_offxyq] |
| vinserti128 m3, [grain_lutq+offxyq+82], 1 |
| vinserti128 m6, [grain_lutq+top_offxyq+8], 1 |
| movd xm4, [grain_lutq+left_offxyq] |
| movd xm7, [grain_lutq+topleft_offxyq] |
| vinserti128 m4, [grain_lutq+left_offxyq+82], 1 |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw m4, m3 |
| punpcklbw xm7, xm6 |
| %if %1 |
| pmaddubsw m4, m9, m4 |
| pmaddubsw xm7, xm9, xm7 |
| pmulhrsw m4, [pw_1024] |
| pmulhrsw xm7, [pw_1024] |
| %else |
| pmaddubsw m4, m15, m4 |
| pmaddubsw xm7, xm15, xm7 |
| pmulhrsw m4, m14 |
| pmulhrsw xm7, xm14 |
| %endif |
| packsswb m4, m4 |
| packsswb xm7, xm7 |
| pcmpeqw m9, m9 ; this is kind of ugly |
| psrldq m9, 15 |
| vpblendvb m3, m3, m4, m9 |
| shufpd m9, m9, m9, 1110b |
| vpblendvb m6, m6, m7, m9 |
| vpermq m9, m3, q3120 |
| ; followed by v interpolation (top | cur -> cur) |
| punpcklbw m6, m9 |
| %if %1 |
| vpbroadcastd m9, [pb_23_22] |
| pmaddubsw m6, m9, m6 |
| pmulhrsw m6, [pw_1024] |
| %else |
| pmaddubsw m6, m15, m6 |
| pmulhrsw m6, m14 |
| %endif |
| packsswb m6, m6 |
| vpermq m6, m6, q3120 |
| vpblendd m3, m3, m6, 00001111b |
| pcmpgtb m7, m2, m3 |
| punpcklbw m2, m3, m7 |
| punpckhbw m3, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmullw m2, m8 |
| pmullw m3, m5 |
| pmulhrsw m2, m11 |
| pmulhrsw m3, m11 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| pmaxsw m0, m13 |
| pmaxsw m1, m13 |
| pminsw m0, m12 |
| pminsw m1, m12 |
| packuswb m0, m1 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*4] |
| add grain_lutq, 82*2 |
| sub hb, 2 |
| jg %%loop_y_h_overlap |
| |
| %%end_y_hv_overlap: |
| add wq, 16 |
| jge %%end_hv |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*2] |
| add srcq, wq |
| add dstq, wq |
| jmp %%loop_x_hv_overlap |
| |
| %%end_hv: |
| RET |
| %endmacro |
| |
| FGUV_32x32xN_LOOP 1 |
| .csfl: |
| FGUV_32x32xN_LOOP 0 |
| |
| %endif ; ARCH_X86_64 |
