| ; Copyright © 2019-2022, VideoLAN and dav1d authors |
| ; Copyright © 2019-2022, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "config.asm" |
| %include "ext/x86/x86inc.asm" |
| %include "x86/filmgrain_common.asm" |
| |
| %if ARCH_X86_64 |
| |
| SECTION_RODATA 32 |
| pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 |
| gen_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 |
| gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 |
| gen_shufB: db 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11 |
| gen_shufC: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 |
| gen_shufD: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 |
| ; note: the order of (some of) the following constants matter |
| pb_27_17: times 2 db 27, 17 |
| byte_blend: db 0, 0, 0, -1 |
| pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 |
| pb_17_27: times 2 db 17, 27 |
| pb_1: times 4 db 1 |
| pb_23_22: db 23, 22, 0, 32, 0, 32, 0, 32 |
| next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 |
| pw_seed_xor: times 2 dw 0xb524 |
| times 2 dw 0x49d8 |
| fg_min: times 4 db 0 |
| times 4 db 16 |
| fg_max: times 4 db 255 |
| times 4 db 240 |
| times 4 db 235 |
| pd_m65536: dd -65536 |
| pw_8: times 2 dw 8 |
| pw_1024: times 2 dw 1024 |
| hmul_bits: dw 32768, 16384, 8192, 4096 |
| round: dw 2048, 1024, 512 |
| mul_bits: dw 256, 128, 64, 32, 16 |
| round_vals: dw 32, 64, 128, 256, 512 |
| pw_1: dw 1 |
| |
| %macro JMP_TABLE 2-* |
| %1_8bpc_%2_table: |
| %xdefine %%base %1_8bpc_%2_table |
| %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) |
| %rep %0 - 2 |
| dd %%prefix %+ .ar%3 - %%base |
| %rotate 1 |
| %endrep |
| %endmacro |
| |
| JMP_TABLE generate_grain_y, avx2, 0, 1, 2, 3 |
| JMP_TABLE generate_grain_uv_420, avx2, 0, 1, 2, 3 |
| JMP_TABLE generate_grain_uv_422, avx2, 0, 1, 2, 3 |
| JMP_TABLE generate_grain_uv_444, avx2, 0, 1, 2, 3 |
| |
| SECTION .text |
| |
| INIT_YMM avx2 |
| cglobal generate_grain_y_8bpc, 2, 9, 8, buf, fg_data |
| %define base r4-generate_grain_y_8bpc_avx2_table |
| lea r4, [generate_grain_y_8bpc_avx2_table] |
| vpbroadcastw xm0, [fg_dataq+FGData.seed] |
| mov r6d, [fg_dataq+FGData.grain_scale_shift] |
| movq xm1, [base+next_upperbit_mask] |
| movsxd r5, [fg_dataq+FGData.ar_coeff_lag] |
| movq xm4, [base+mul_bits] |
| movq xm5, [base+hmul_bits] |
| mov r7, -73*82 |
| mova xm6, [base+pb_mask] |
| sub bufq, r7 |
| vpbroadcastw xm7, [base+round+r6*2] |
| lea r6, [gaussian_sequence] |
| movsxd r5, [r4+r5*4] |
| .loop: |
| pand xm2, xm0, xm1 |
| psrlw xm3, xm2, 10 |
| por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set |
| pmullw xm2, xm4 ; bits 0x0f00 are set |
| pmulhuw xm0, xm5 |
| pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds |
| psllq xm2, xm3, 30 |
| por xm2, xm3 |
| psllq xm3, xm2, 15 |
| por xm2, xm0 ; aggregate each bit into next seed's high bit |
| por xm3, xm2 ; 4 next output seeds |
| pshuflw xm0, xm3, q3333 |
| psrlw xm3, 5 |
| pand xm2, xm0, xm1 |
| movq r2, xm3 |
| psrlw xm3, xm2, 10 |
| por xm2, xm3 |
| pmullw xm2, xm4 |
| pmulhuw xm0, xm5 |
| movzx r3d, r2w |
| pshufb xm3, xm6, xm2 |
| psllq xm2, xm3, 30 |
| por xm2, xm3 |
| psllq xm3, xm2, 15 |
| por xm0, xm2 |
| movd xm2, [r6+r3*2] |
| rorx r3, r2, 32 |
| por xm3, xm0 |
| shr r2d, 16 |
| pinsrw xm2, [r6+r2*2], 1 |
| pshuflw xm0, xm3, q3333 |
| movzx r2d, r3w |
| psrlw xm3, 5 |
| pinsrw xm2, [r6+r2*2], 2 |
| shr r3d, 16 |
| movq r2, xm3 |
| pinsrw xm2, [r6+r3*2], 3 |
| movzx r3d, r2w |
| pinsrw xm2, [r6+r3*2], 4 |
| rorx r3, r2, 32 |
| shr r2d, 16 |
| pinsrw xm2, [r6+r2*2], 5 |
| movzx r2d, r3w |
| pinsrw xm2, [r6+r2*2], 6 |
| shr r3d, 16 |
| pinsrw xm2, [r6+r3*2], 7 |
| pmulhrsw xm2, xm7 |
| packsswb xm2, xm2 |
| movq [bufq+r7], xm2 |
| add r7, 8 |
| jl .loop |
| |
| ; auto-regression code |
| add r5, r4 |
| jmp r5 |
| |
| .ar1: |
| DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] |
| movd xm5, [fg_dataq+FGData.ar_coeffs_y] |
| mova xm2, [base+gen_shufC] |
| DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0 |
| pinsrb xm5, [base+pb_1], 3 |
| vpbroadcastw xm3, [base+round_vals+shiftq*2-12] ; rnd |
| pmovsxbw xm5, xm5 |
| pshufd xm4, xm5, q0000 |
| pshufd xm5, xm5, q1111 |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| mov mind, -128 |
| mov maxd, 127 |
| .y_loop_ar1: |
| mov xq, -76 |
| movsx val3d, byte [bufq+xq-1] |
| .x_loop_ar1: |
| pmovsxbw xm1, [bufq+xq-82-3] |
| pshufb xm0, xm1, xm2 |
| punpckhwd xm1, xm3 |
| pmaddwd xm0, xm4 |
| pmaddwd xm1, xm5 |
| paddd xm0, xm1 |
| .x_loop_ar1_inner: |
| movd val0d, xm0 |
| psrldq xm0, 4 |
| imul val3d, cf3d |
| add val3d, val0d |
| movsx val0d, byte [bufq+xq] |
| sarx val3d, val3d, shiftd |
| add val3d, val0d |
| cmp val3d, maxd |
| cmovns val3d, maxd |
| cmp val3d, mind |
| cmovs val3d, mind |
| mov [bufq+xq], val3b |
| ; keep val3d in-place as left for next x iteration |
| inc xq |
| jz .x_loop_ar1_end |
| test xb, 3 |
| jnz .x_loop_ar1_inner |
| jmp .x_loop_ar1 |
| .x_loop_ar1_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar1 |
| .ar0: |
| RET |
| |
| .ar2: |
| %if WIN64 |
| ; xmm6 and xmm7 already saved |
| %assign xmm_regs_used 16 |
| %assign stack_size_padded 168 |
| SUB rsp, stack_size_padded |
| movaps [rsp+16*2], xmm8 |
| movaps [rsp+16*3], xmm9 |
| movaps [rsp+16*4], xmm10 |
| movaps [rsp+16*5], xmm11 |
| movaps [rsp+16*6], xmm12 |
| movaps [rsp+16*7], xmm13 |
| movaps [rsp+16*8], xmm14 |
| movaps [rsp+16*9], xmm15 |
| %endif |
| DEFINE_ARGS buf, fg_data, h, x |
| mov r6d, [fg_dataq+FGData.ar_coeff_shift] |
| pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7 |
| movd xm9, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11 |
| vpbroadcastd xm10, [base+round_vals-14+r6*2] |
| movd xm11, [base+byte_blend+1] |
| pmovsxbw xm9, xm9 |
| pshufd xm4, xm7, q0000 |
| mova xm12, [base+gen_shufA] |
| pshufd xm5, xm7, q3333 |
| mova xm13, [base+gen_shufB] |
| pshufd xm6, xm7, q1111 |
| mova xm14, [base+gen_shufC] |
| pshufd xm7, xm7, q2222 |
| mova xm15, [base+gen_shufD] |
| pshufd xm8, xm9, q0000 |
| psrld xm10, 16 |
| pshufd xm9, xm9, q1111 |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| .y_loop_ar2: |
| mov xq, -76 |
| .x_loop_ar2: |
| pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] |
| pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] |
| pshufb xm2, xm0, xm12 |
| pmaddwd xm2, xm4 |
| pshufb xm3, xm1, xm13 |
| pmaddwd xm3, xm5 |
| paddd xm2, xm3 |
| pshufb xm3, xm0, xm14 |
| pmaddwd xm3, xm6 |
| punpckhqdq xm0, xm0 |
| punpcklwd xm0, xm1 |
| pmaddwd xm0, xm7 |
| pshufb xm1, xm15 |
| pmaddwd xm1, xm8 |
| paddd xm2, xm10 |
| paddd xm2, xm3 |
| paddd xm0, xm1 |
| paddd xm2, xm0 |
| movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] |
| .x_loop_ar2_inner: |
| pmovsxbw xm1, xm0 |
| pmaddwd xm3, xm9, xm1 |
| psrldq xm1, 4 ; y=0,x=0 |
| paddd xm3, xm2 |
| psrldq xm2, 4 ; shift top to next pixel |
| psrad xm3, [fg_dataq+FGData.ar_coeff_shift] |
| ; don't packssdw since we only care about one value |
| paddw xm3, xm1 |
| packsswb xm3, xm3 |
| pextrb [bufq+xq], xm3, 0 |
| pslldq xm3, 2 |
| vpblendvb xm0, xm3, xm11 |
| psrldq xm0, 1 |
| inc xq |
| jz .x_loop_ar2_end |
| test xb, 3 |
| jnz .x_loop_ar2_inner |
| jmp .x_loop_ar2 |
| .x_loop_ar2_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar2 |
| RET |
| |
| INIT_YMM avx2 |
| .ar3: |
| %if WIN64 |
| ; xmm6 and xmm7 already saved |
| %assign stack_offset 16 |
| ALLOC_STACK 16*14 |
| %assign stack_size stack_size - 16*4 |
| %assign xmm_regs_used 12 |
| movaps [rsp+16*12], xmm8 |
| movaps [rsp+16*13], xmm9 |
| movaps [rsp+16*14], xmm10 |
| movaps [rsp+16*15], xmm11 |
| %else |
| ALLOC_STACK 16*12 |
| %endif |
| mov r6d, [fg_dataq+FGData.ar_coeff_shift] |
| movd xm11, [base+byte_blend] |
| pmovsxbw m1, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 |
| pmovsxbw xm2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 |
| pshufd m0, m1, q0000 |
| mova [rsp+16* 0], m0 |
| pshufd m0, m1, q1111 |
| mova [rsp+16* 2], m0 |
| pshufd m0, m1, q2222 |
| mova [rsp+16* 4], m0 |
| pshufd m1, m1, q3333 |
| mova [rsp+16* 6], m1 |
| pshufd xm0, xm2, q0000 |
| mova [rsp+16* 8], xm0 |
| pshufd xm0, xm2, q1111 |
| mova [rsp+16* 9], xm0 |
| psrldq xm7, xm2, 10 |
| mova m8, [base+gen_shufA] |
| pinsrw xm2, [base+pw_1], 5 |
| mova m9, [base+gen_shufC] |
| pshufd xm2, xm2, q2222 |
| movu m10, [base+gen_shufE] |
| vpbroadcastw xm6, [base+round_vals-12+r6*2] |
| pinsrw xm7, [base+round_vals+r6*2-10], 3 |
| mova [rsp+16*10], xm2 |
| DEFINE_ARGS buf, fg_data, h, x |
| sub bufq, 82*73-(82*3+79) |
| mov hd, 70 |
| .y_loop_ar3: |
| mov xq, -76 |
| .x_loop_ar3: |
| movu xm5, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] |
| vinserti128 m5, [bufq+xq-82*2-3], 1 ; y=-2,x=[-3,+12] |
| movu xm4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] |
| punpcklbw m3, m5, m5 |
| punpckhwd m5, m4 |
| psraw m3, 8 |
| punpcklbw m5, m5 |
| psraw m5, 8 |
| punpcklbw xm4, xm4 |
| psraw xm4, 8 |
| pshufb m0, m3, m8 |
| pmaddwd m0, [rsp+16*0] |
| pshufb m1, m3, m9 |
| pmaddwd m1, [rsp+16*2] |
| shufps m2, m3, m5, q1032 |
| paddd m0, m1 |
| pshufb m1, m2, m8 |
| vperm2i128 m3, m4, 0x21 |
| pmaddwd m1, [rsp+16*4] |
| shufps xm2, xm3, q1021 |
| vpblendd m2, m3, 0xf0 |
| pshufb m2, m10 |
| paddd m0, m1 |
| pmaddwd m2, [rsp+16*6] |
| pshufb xm1, xm4, xm9 |
| pmaddwd xm1, [rsp+16*8] |
| shufps xm4, xm5, q1132 |
| paddd m0, m2 |
| pshufb xm2, xm4, xm8 |
| pshufd xm4, xm4, q2121 |
| pmaddwd xm2, [rsp+16*9] |
| punpcklwd xm4, xm6 |
| pmaddwd xm4, [rsp+16*10] |
| vextracti128 xm3, m0, 1 |
| paddd xm0, xm1 |
| movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] |
| paddd xm2, xm4 |
| paddd xm0, xm2 |
| paddd xm0, xm3 |
| .x_loop_ar3_inner: |
| pmovsxbw xm2, xm1 |
| pmaddwd xm2, xm7 |
| pshufd xm3, xm2, q1111 |
| paddd xm2, xm0 ; add top |
| paddd xm2, xm3 ; left+cur |
| psrldq xm0, 4 |
| psrad xm2, [fg_dataq+FGData.ar_coeff_shift] |
| ; don't packssdw since we only care about one value |
| packsswb xm2, xm2 |
| pextrb [bufq+xq], xm2, 0 |
| pslldq xm2, 3 |
| vpblendvb xm1, xm2, xm11 |
| psrldq xm1, 1 |
| inc xq |
| jz .x_loop_ar3_end |
| test xb, 3 |
| jnz .x_loop_ar3_inner |
| jmp .x_loop_ar3 |
| .x_loop_ar3_end: |
| add bufq, 82 |
| dec hd |
| jg .y_loop_ar3 |
| RET |
| |
| %macro GEN_GRAIN_UV_FN 3 ; ss_name, ss_x, ss_y |
| INIT_XMM avx2 |
| cglobal generate_grain_uv_%1_8bpc, 4, 10, 16, buf, bufy, fg_data, uv |
| %define base r4-generate_grain_uv_%1_8bpc_avx2_table |
| lea r4, [generate_grain_uv_%1_8bpc_avx2_table] |
| vpbroadcastw xm0, [fg_dataq+FGData.seed] |
| mov r6d, [fg_dataq+FGData.grain_scale_shift] |
| movq xm1, [base+next_upperbit_mask] |
| movq xm4, [base+mul_bits] |
| movq xm5, [base+hmul_bits] |
| mova xm6, [base+pb_mask] |
| vpbroadcastw xm7, [base+round+r6*2] |
| vpbroadcastd xm2, [base+pw_seed_xor+uvq*4] |
| pxor xm0, xm2 |
| lea r6, [gaussian_sequence] |
| %if %2 |
| mov r7d, 73-35*%3 |
| add bufq, 44 |
| .loop_y: |
| mov r5, -44 |
| %else |
| mov r5, -73*82 |
| sub bufq, r5 |
| %endif |
| .loop: |
| pand xm2, xm0, xm1 |
| psrlw xm3, xm2, 10 |
| por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set |
| pmullw xm2, xm4 ; bits 0x0f00 are set |
| pmulhuw xm0, xm5 |
| pshufb xm3, xm6, xm2 ; set 15th bit for next 4 seeds |
| psllq xm2, xm3, 30 |
| por xm2, xm3 |
| psllq xm3, xm2, 15 |
| por xm2, xm0 ; aggregate each bit into next seed's high bit |
| por xm2, xm3 ; 4 next output seeds |
| pshuflw xm0, xm2, q3333 |
| psrlw xm2, 5 |
| movq r8, xm2 |
| movzx r9d, r8w |
| movd xm2, [r6+r9*2] |
| rorx r9, r8, 32 |
| shr r8d, 16 |
| pinsrw xm2, [r6+r8*2], 1 |
| movzx r8d, r9w |
| pinsrw xm2, [r6+r8*2], 2 |
| shr r9d, 16 |
| pinsrw xm2, [r6+r9*2], 3 |
| pmulhrsw xm2, xm7 |
| packsswb xm2, xm2 |
| movd [bufq+r5], xm2 |
| add r5, 4 |
| jl .loop |
| %if %2 |
| add bufq, 82 |
| dec r7d |
| jg .loop_y |
| %endif |
| |
| ; auto-regression code |
| movsxd r6, [fg_dataq+FGData.ar_coeff_lag] |
| movsxd r6, [base+generate_grain_uv_%1_8bpc_avx2_table+r6*4] |
| add r6, r4 |
| jmp r6 |
| |
| INIT_YMM avx2 |
| .ar0: |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| imul uvd, 28 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movd xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq] |
| movd xm3, [base+hmul_bits+shiftq*2] |
| DEFINE_ARGS buf, bufy, h |
| pmovsxbw xm2, xm2 |
| %if %2 |
| vpbroadcastd m7, [base+pb_1] |
| vpbroadcastw m6, [base+hmul_bits+2+%3*2] |
| %endif |
| vpbroadcastw m2, xm2 |
| vpbroadcastw m3, xm3 |
| pxor m12, m12 |
| %if %2 |
| sub bufq, 82*(73-35*%3)+82-(82*3+41) |
| %else |
| sub bufq, 82*70-3 |
| %endif |
| add bufyq, 3+82*3 |
| mov hd, 70-35*%3 |
| .y_loop_ar0: |
| %if %2 |
| ; first 32 pixels |
| movu xm4, [bufyq] |
| vinserti128 m4, [bufyq+32], 1 |
| %if %3 |
| movu xm0, [bufyq+82] |
| vinserti128 m0, [bufyq+82+32], 1 |
| %endif |
| movu xm5, [bufyq+16] |
| vinserti128 m5, [bufyq+48], 1 |
| %if %3 |
| movu xm1, [bufyq+82+16] |
| vinserti128 m1, [bufyq+82+48], 1 |
| %endif |
| pmaddubsw m4, m7, m4 |
| %if %3 |
| pmaddubsw m0, m7, m0 |
| %endif |
| pmaddubsw m5, m7, m5 |
| %if %3 |
| pmaddubsw m1, m7, m1 |
| paddw m4, m0 |
| paddw m5, m1 |
| %endif |
| pmulhrsw m4, m6 |
| pmulhrsw m5, m6 |
| %else |
| xor r3d, r3d |
| ; first 32x2 pixels |
| .x_loop_ar0: |
| movu m4, [bufyq+r3] |
| pcmpgtb m0, m12, m4 |
| punpckhbw m5, m4, m0 |
| punpcklbw m4, m0 |
| %endif |
| pmullw m4, m2 |
| pmullw m5, m2 |
| pmulhrsw m4, m3 |
| pmulhrsw m5, m3 |
| %if %2 |
| movu m1, [bufq] |
| %else |
| movu m1, [bufq+r3] |
| %endif |
| pcmpgtb m8, m12, m1 |
| punpcklbw m0, m1, m8 |
| punpckhbw m1, m8 |
| paddw m0, m4 |
| paddw m1, m5 |
| packsswb m0, m1 |
| %if %2 |
| movu [bufq], m0 |
| %else |
| movu [bufq+r3], m0 |
| add r3d, 32 |
| cmp r3d, 64 |
| jl .x_loop_ar0 |
| %endif |
| |
| ; last 6/12 pixels |
| movu xm4, [bufyq+32*2] |
| %if %2 |
| %if %3 |
| movu xm5, [bufyq+32*2+82] |
| %endif |
| pmaddubsw xm4, xm7, xm4 |
| %if %3 |
| pmaddubsw xm5, xm7, xm5 |
| paddw xm4, xm5 |
| %endif |
| movq xm0, [bufq+32] |
| pmulhrsw xm4, xm6 |
| pmullw xm4, xm2 |
| pmulhrsw xm4, xm3 |
| pcmpgtb xm5, xm12, xm0 |
| punpcklbw xm5, xm0, xm5 |
| paddw xm4, xm5 |
| packsswb xm4, xm4 |
| pblendw xm0, xm4, xm0, 1000b |
| movq [bufq+32], xm0 |
| %else |
| movu xm0, [bufq+64] |
| pcmpgtb xm1, xm12, xm4 |
| punpckhbw xm5, xm4, xm1 |
| punpcklbw xm4, xm1 |
| pmullw xm5, xm2 |
| pmullw xm4, xm2 |
| vpblendd xm1, xm3, xm12, 0x0c |
| pmulhrsw xm5, xm1 |
| pmulhrsw xm4, xm3 |
| pcmpgtb xm1, xm12, xm0 |
| punpckhbw xm8, xm0, xm1 |
| punpcklbw xm0, xm1 |
| paddw xm5, xm8 |
| paddw xm0, xm4 |
| packsswb xm0, xm5 |
| movu [bufq+64], xm0 |
| %endif |
| add bufq, 82 |
| add bufyq, 82<<%3 |
| dec hd |
| jg .y_loop_ar0 |
| RET |
| |
| INIT_XMM avx2 |
| .ar1: |
| DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift |
| imul uvd, 28 |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] |
| movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] |
| pinsrb xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 3 |
| DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift |
| pmovsxbw xm4, xm4 |
| pshufd xm5, xm4, q1111 |
| pshufd xm4, xm4, q0000 |
| pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd |
| %if %2 |
| vpbroadcastd xm7, [base+pb_1] |
| vpbroadcastw xm6, [base+hmul_bits+2+%3*2] |
| %endif |
| vpbroadcastd xm3, xm3 |
| %if %2 |
| sub bufq, 82*(73-35*%3)+44-(82*3+41) |
| %else |
| sub bufq, 82*70-(82-3) |
| %endif |
| add bufyq, 79+82*3 |
| mov hd, 70-35*%3 |
| mov mind, -128 |
| mov maxd, 127 |
| .y_loop_ar1: |
| mov xq, -(76>>%2) |
| movsx val3d, byte [bufq+xq-1] |
| .x_loop_ar1: |
| pmovsxbw xm0, [bufq+xq-82-1] ; top/left |
| %if %2 |
| movq xm8, [bufyq+xq*2] |
| %if %3 |
| movq xm9, [bufyq+xq*2+82] |
| %endif |
| %endif |
| psrldq xm2, xm0, 2 ; top |
| psrldq xm1, xm0, 4 ; top/right |
| %if %2 |
| pmaddubsw xm8, xm7, xm8 |
| %if %3 |
| pmaddubsw xm9, xm7, xm9 |
| paddw xm8, xm9 |
| %endif |
| pmulhrsw xm8, xm6 |
| %else |
| pmovsxbw xm8, [bufyq+xq] |
| %endif |
| punpcklwd xm0, xm2 |
| punpcklwd xm1, xm8 |
| pmaddwd xm0, xm4 |
| pmaddwd xm1, xm5 |
| paddd xm0, xm1 |
| paddd xm0, xm3 |
| .x_loop_ar1_inner: |
| movd val0d, xm0 |
| psrldq xm0, 4 |
| imul val3d, cf3d |
| add val3d, val0d |
| sarx val3d, val3d, shiftd |
| movsx val0d, byte [bufq+xq] |
| add val3d, val0d |
| cmp val3d, maxd |
| cmovns val3d, maxd |
| cmp val3d, mind |
| cmovs val3d, mind |
| mov byte [bufq+xq], val3b |
| ; keep val3d in-place as left for next x iteration |
| inc xq |
| jz .x_loop_ar1_end |
| test xq, 3 |
| jnz .x_loop_ar1_inner |
| jmp .x_loop_ar1 |
| |
| .x_loop_ar1_end: |
| add bufq, 82 |
| add bufyq, 82<<%3 |
| dec hd |
| jg .y_loop_ar1 |
| RET |
| |
| .ar2: |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| imul uvd, 28 |
| vpbroadcastw xm13, [base+round_vals-12+shiftq*2] |
| pmovsxbw xm7, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 |
| pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 |
| pinsrw xm0, [base+pw_1], 5 |
| %if %2 |
| vpbroadcastw xm12, [base+hmul_bits+2+%3*2] |
| vpbroadcastd xm11, [base+pb_1] |
| %endif |
| DEFINE_ARGS buf, bufy, fg_data, h, unused, x |
| pshufd xm4, xm7, q0000 |
| pshufd xm5, xm7, q3333 |
| pshufd xm6, xm7, q1111 |
| pshufd xm7, xm7, q2222 |
| pshufd xm8, xm0, q0000 |
| pshufd xm9, xm0, q1111 |
| pshufd xm10, xm0, q2222 |
| %if %2 |
| sub bufq, 82*(73-35*%3)+44-(82*3+41) |
| %else |
| sub bufq, 82*70-(82-3) |
| %endif |
| add bufyq, 79+82*3 |
| mov hd, 70-35*%3 |
| .y_loop_ar2: |
| mov xq, -(76>>%2) |
| |
| .x_loop_ar2: |
| pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] |
| pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] |
| pshufb xm2, xm0, [base+gen_shufA] |
| pmaddwd xm2, xm4 |
| pshufb xm3, xm1, [base+gen_shufB] |
| pmaddwd xm3, xm5 |
| paddd xm2, xm3 |
| pshufb xm3, xm0, [base+gen_shufC] |
| pmaddwd xm3, xm6 |
| punpckhqdq xm0, xm0 ; y=-2,x=[+2,+5] |
| punpcklwd xm0, xm1 |
| pmaddwd xm0, xm7 |
| pshufb xm1, [gen_shufD] |
| pmaddwd xm1, xm8 |
| paddd xm2, xm3 |
| paddd xm0, xm1 |
| paddd xm2, xm0 |
| |
| %if %2 |
| movq xm0, [bufyq+xq*2] |
| %if %3 |
| movq xm3, [bufyq+xq*2+82] |
| %endif |
| pmaddubsw xm0, xm11, xm0 |
| %if %3 |
| pmaddubsw xm3, xm11, xm3 |
| paddw xm0, xm3 |
| %endif |
| pmulhrsw xm0, xm12 |
| %else |
| pmovsxbw xm0, [bufyq+xq] |
| %endif |
| punpcklwd xm0, xm13 |
| pmaddwd xm0, xm10 |
| paddd xm2, xm0 |
| |
| movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] |
| .x_loop_ar2_inner: |
| pmovsxbw xm0, xm0 |
| pmaddwd xm3, xm0, xm9 |
| psrldq xm0, 2 |
| paddd xm3, xm2 |
| psrldq xm2, 4 ; shift top to next pixel |
| psrad xm3, [fg_dataq+FGData.ar_coeff_shift] |
| pslldq xm3, 2 |
| paddw xm3, xm0 |
| pblendw xm0, xm3, 00000010b |
| packsswb xm0, xm0 |
| pextrb [bufq+xq], xm0, 1 |
| inc xq |
| jz .x_loop_ar2_end |
| test xb, 3 |
| jnz .x_loop_ar2_inner |
| jmp .x_loop_ar2 |
| |
| .x_loop_ar2_end: |
| add bufq, 82 |
| add bufyq, 82<<%3 |
| dec hd |
| jg .y_loop_ar2 |
| RET |
| |
| INIT_YMM avx2 |
| .ar3: |
| DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift |
| mov shiftd, [fg_dataq+FGData.ar_coeff_shift] |
| imul uvd, 28 |
| pmovsxbw m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15 |
| pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-23 |
| vpbroadcastb xm2, [fg_dataq+FGData.ar_coeffs_uv+uvq+24] ; cf24 [luma] |
| movd xm13, [base+round_vals-10+shiftq*2] |
| vpbroadcastd xm14, [base+round_vals-14+shiftq*2] |
| pshufd m6, m0, q0000 |
| pshufd m7, m0, q1111 |
| pshufd m8, m0, q2222 |
| pshufd m9, m0, q3333 |
| pshufd xm10, xm1, q0000 |
| pshufd xm11, xm1, q1111 |
| pshufhw xm12, xm1, q0000 |
| psraw xm2, 8 |
| palignr xm13, xm1, 10 |
| punpckhwd xm12, xm2 ; interleave luma cf |
| psrld xm14, 16 |
| DEFINE_ARGS buf, bufy, fg_data, h, unused, x |
| %if %2 |
| vpbroadcastw xm15, [base+hmul_bits+2+%3*2] |
| sub bufq, 82*(73-35*%3)+44-(82*3+41) |
| %else |
| sub bufq, 82*70-(82-3) |
| %endif |
| add bufyq, 79+82*3 |
| mov hd, 70-35*%3 |
| .y_loop_ar3: |
| mov xq, -(76>>%2) |
| .x_loop_ar3: |
| vbroadcasti128 m3, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12 |
| palignr xm1, xm3, [bufq+xq-82*3-9], 6 ; y=-3,x=[-3,+12] |
| vbroadcasti128 m4, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] |
| vpblendd m3, m1, 0x0f |
| pxor m0, m0 |
| pcmpgtb m2, m0, m3 |
| pcmpgtb m0, m4 |
| punpcklbw m1, m3, m2 |
| punpckhbw m3, m2 |
| punpcklbw m2, m4, m0 |
| punpckhbw xm4, xm0 |
| pshufb m0, m1, [base+gen_shufA] |
| pmaddwd m0, m6 |
| pshufb m5, m1, [base+gen_shufC] |
| pmaddwd m5, m7 |
| shufps m1, m3, q1032 |
| paddd m0, m5 |
| pshufb m5, m1, [base+gen_shufA] |
| pmaddwd m5, m8 |
| shufps xm1, xm3, q2121 |
| vpblendd m1, m2, 0xf0 |
| pshufb m1, [base+gen_shufE] |
| pmaddwd m1, m9 |
| paddd m0, m5 |
| pshufb xm3, xm2, [base+gen_shufC] |
| paddd m0, m1 |
| pmaddwd xm3, xm10 |
| palignr xm1, xm4, xm2, 2 |
| punpckhwd xm1, xm2, xm1 |
| pmaddwd xm1, xm11 |
| palignr xm4, xm2, 12 |
| paddd xm3, xm1 |
| %if %2 |
| vpbroadcastd xm5, [base+pb_1] |
| movq xm1, [bufyq+xq*2] |
| pmaddubsw xm1, xm5, xm1 |
| %if %3 |
| movq xm2, [bufyq+xq*2+82] |
| pmaddubsw xm5, xm2 |
| paddw xm1, xm5 |
| %endif |
| pmulhrsw xm1, xm15 |
| %else |
| pmovsxbw xm1, [bufyq+xq] |
| %endif |
| punpcklwd xm4, xm1 |
| pmaddwd xm4, xm12 |
| movq xm1, [bufq+xq-3] ; y=0,x=[-3,+4] |
| vextracti128 xm2, m0, 1 |
| paddd xm0, xm14 |
| paddd xm3, xm4 |
| paddd xm0, xm3 |
| paddd xm0, xm2 |
| .x_loop_ar3_inner: |
| pmovsxbw xm1, xm1 |
| pmaddwd xm2, xm13, xm1 |
| pshuflw xm3, xm2, q1032 |
| paddd xm2, xm0 ; add top |
| paddd xm2, xm3 ; left+cur |
| psrldq xm0, 4 |
| psrad xm2, [fg_dataq+FGData.ar_coeff_shift] |
| psrldq xm1, 2 |
| ; don't packssdw, we only care about one value |
| punpckldq xm2, xm2 |
| pblendw xm1, xm2, 0100b |
| packsswb xm1, xm1 |
| pextrb [bufq+xq], xm1, 2 |
| inc xq |
| jz .x_loop_ar3_end |
| test xb, 3 |
| jnz .x_loop_ar3_inner |
| jmp .x_loop_ar3 |
| .x_loop_ar3_end: |
| add bufq, 82 |
| add bufyq, 82<<%3 |
| dec hd |
| jg .y_loop_ar3 |
| RET |
| %endmacro |
| |
| INIT_YMM avx2 |
| cglobal fgy_32x32xn_8bpc, 6, 13, 15, dst, src, stride, fg_data, w, scaling, \ |
| grain_lut, h, sby, see, overlap |
| %define base r9-pd_m65536 |
| lea r9, [pd_m65536] |
| mov r6d, [fg_dataq+FGData.scaling_shift] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| mov sbyd, sbym |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| vpbroadcastd m8, [base+pd_m65536] |
| vpbroadcastw m9, [base+mul_bits+r6*2-14] |
| vpbroadcastd m10, [base+fg_min+r7*4] |
| vpbroadcastd m11, [base+fg_max+r7*8] |
| vpbroadcastd m12, [base+pw_1024] |
| movq xm13, [base+pb_27_17_17_27] |
| test sbyd, sbyd |
| setnz r7b |
| pxor m7, m7 |
| test r7b, overlapb |
| jnz .vertical_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rorx seed, seed, 24 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| |
| .loop_x: |
| rorx r6, seeq, 1 |
| or seed, 0xEFF4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offyd, [offyq+offxq*2+747] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, overlap |
| |
| mov hd, hm |
| mov grain_lutq, grain_lutmp |
| .loop_y: |
| ; src |
| mova m2, [srcq] |
| punpcklbw m0, m2, m7 |
| punpckhbw m1, m2, m7 |
| |
| ; scaling[src] |
| pandn m4, m8, m0 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, m0, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m1 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| pblendw m2, m4, 0xaa |
| psrld m4, m1, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m4-2], m6 |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m5, [grain_lutq+offxyq] |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hd |
| jg .loop_y |
| |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| test overlapd, overlapd |
| jz .loop_x |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne .loop_x_hv_overlap |
| |
| ; horizontal overlap (without vertical overlap) |
| .loop_x_h_overlap: |
| rorx r6, seeq, 1 |
| or seed, 0xEFF4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy |
| |
| lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offyd, [offyq+offxq*2+747] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| .loop_y_h_overlap: |
| ; src |
| mova m2, [srcq] |
| punpcklbw m0, m2, m7 |
| punpckhbw m1, m2, m7 |
| |
| ; scaling[src] |
| pandn m4, m8, m0 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, m0, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m1 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| pblendw m2, m4, 0xaa |
| psrld m4, m1, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m4-2], m6 |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m5, [grain_lutq+offxyq] |
| movd xm4, [grain_lutq+left_offxyq] |
| punpcklbw xm4, xm5 |
| pmaddubsw xm4, xm13, xm4 |
| pmulhrsw xm4, xm12 |
| packsswb xm4, xm4 |
| vpblendd m4, m5, 0xfe |
| punpckhbw m5, m7 |
| punpcklbw m4, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hd |
| jg .loop_y_h_overlap |
| |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne .loop_x_hv_overlap |
| jmp .loop_x_h_overlap |
| |
| .vertical_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ |
| unused, sby, see, overlap |
| |
| movzx sbyd, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, sbyd, 173 * 0x00010001 |
| imul sbyd, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add sbyd, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and sbyd, 0xff00ff00 |
| xor seed, r7d |
| xor seed, sbyd ; (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| |
| .loop_x_v_overlap: |
| vpbroadcastd m14, [pb_27_17] |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*2+0x10001*747+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, top_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| .loop_y_v_overlap: |
| ; src |
| mova m2, [srcq] |
| punpcklbw m0, m2, m7 |
| punpckhbw m1, m2, m7 |
| |
| ; scaling[src] |
| pandn m4, m8, m0 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, m0, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m1 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| pblendw m2, m4, 0xaa |
| psrld m4, m1, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m4-2], m6 |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m6, [grain_lutq+offxyq] |
| movu m4, [grain_lutq+top_offxyq] |
| punpcklbw m5, m4, m6 |
| punpckhbw m4, m6 |
| pmaddubsw m5, m14, m5 |
| pmaddubsw m4, m14, m4 |
| pmulhrsw m5, m12 |
| pmulhrsw m4, m12 |
| packsswb m5, m4 |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hb |
| jz .end_y_v_overlap |
| vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line |
| ; 2 lines get vertical overlap, then fall back to non-overlap code for |
| ; remaining (up to) 30 lines |
| add hd, 0x80000000 |
| jnc .loop_y_v_overlap |
| jmp .loop_y |
| .end_y_v_overlap: |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| |
| ; since fg_dataq.overlap is guaranteed to be set, we never jump |
| ; back to .loop_x_v_overlap, and instead always fall-through to |
| ; h+v overlap |
| .loop_x_hv_overlap: |
| vpbroadcastd m14, [pb_27_17] |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, top_offxy, topleft_offxy |
| |
| lea topleft_offxyd, [top_offxyq+32] |
| lea left_offxyd, [offyq+32] |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*2+0x10001*747+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, top_offxy, topleft_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| .loop_y_hv_overlap: |
| ; src |
| mova m2, [srcq] |
| punpcklbw m0, m2, m7 |
| punpckhbw m1, m2, m7 |
| |
| ; scaling[src] |
| pandn m4, m8, m0 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, m0, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m1 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| pblendw m2, m4, 0xaa |
| psrld m4, m1, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m4-2], m6 |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| movu m6, [grain_lutq+offxyq] |
| movd xm7, [grain_lutq+left_offxyq] |
| movu m4, [grain_lutq+top_offxyq] |
| movd xm5, [grain_lutq+topleft_offxyq] |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw xm7, xm6 |
| punpcklbw xm5, xm4 |
| pmaddubsw xm7, xm13, xm7 |
| pmaddubsw xm5, xm13, xm5 |
| pmulhrsw xm7, xm12 |
| pmulhrsw xm5, xm12 |
| packsswb xm7, xm7 |
| packsswb xm5, xm5 |
| vpblendd m7, m6, 0xfe |
| vpblendd m5, m4, 0xfe |
| ; followed by v interpolation (top | cur -> cur) |
| punpckhbw m4, m6 |
| punpcklbw m5, m7 |
| pmaddubsw m4, m14, m4 |
| pmaddubsw m5, m14, m5 |
| pmulhrsw m4, m12 |
| pmulhrsw m5, m12 |
| pxor m7, m7 |
| packsswb m5, m4 |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| mova [dstq+srcq], m0 |
| |
| add srcq, strideq |
| add grain_lutq, 82 |
| dec hb |
| jz .end_y_hv_overlap |
| vpbroadcastd m14, [pb_17_27] ; swap weights for second v-overlap line |
| ; 2 lines get vertical overlap, then fall back to non-overlap code for |
| ; remaining (up to) 30 lines |
| add hd, 0x80000000 |
| jnc .loop_y_hv_overlap |
| jmp .loop_y_h_overlap |
| .end_y_hv_overlap: |
| add wq, 32 |
| lea srcq, [src_bakq+wq] |
| jl .loop_x_hv_overlap |
| .end: |
| RET |
| |
| %macro FGUV_FN 3 ; name, ss_hor, ss_ver |
| cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ |
| grain_lut, h, sby, luma, overlap, uv_pl, is_id |
| %define base r11-pd_m65536 |
| lea r11, [pd_m65536] |
| mov r6d, [fg_dataq+FGData.scaling_shift] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| mov r9d, is_idm |
| mov sbyd, sbym |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| vpbroadcastd m8, [base+pd_m65536] |
| vpbroadcastw m9, [base+mul_bits+r6*2-14] |
| vpbroadcastd m10, [base+fg_min+r7*4] |
| shlx r7d, r7d, r9d |
| vpbroadcastd m11, [base+fg_max+r7*4] |
| vpbroadcastd m12, [base+pw_1024] |
| pxor m7, m7 |
| test sbyd, sbyd |
| setnz r7b |
| cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 |
| jne .csfl |
| |
| %macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ |
| h, sby, see, overlap, uv_pl |
| %if %1 |
| mov r6d, uv_plm |
| vpbroadcastd m0, [base+pw_8] |
| vbroadcasti128 m14, [fg_dataq+FGData.uv_mult+r6*4] |
| vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] |
| pshufb m14, m0 ; uv_luma_mult, uv_mult |
| %elif %2 |
| vpbroadcastq m15, [base+pb_23_22] |
| %else |
| vpbroadcastq xm15, [base+pb_27_17_17_27] |
| %endif |
| %if %3 |
| vpbroadcastw m13, [base+pb_23_22] |
| %elif %2 |
| pshufd m13, [base+pb_27_17], q0000 ; 8x27_17, 8x17_27 |
| %endif |
| test r7b, overlapb |
| jnz %%vertical_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rorx seed, seed, 24 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| unused2, unused3, see, overlap, unused4, unused5, lstride |
| |
| mov lumaq, r9mp |
| lea r12, [srcq+wq] |
| lea r13, [dstq+wq] |
| lea r14, [lumaq+wq*(1+%2)] |
| mov r11mp, r12 |
| mov r12mp, r13 |
| mov lstrideq, r10mp |
| neg wq |
| |
| %%loop_x: |
| rorx r6, seeq, 1 |
| or seed, 0xEFF4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, overlap, unused1, unused2, lstride |
| |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164>>%3 |
| lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, unused1, unused2, lstride |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| %%loop_y: |
| ; src |
| %if %2 |
| mova xm3, [lumaq+lstrideq*0+ 0] |
| vinserti128 m3, [lumaq+lstrideq*(1+%3) +0], 1 |
| vpbroadcastd m2, [pb_1] |
| mova xm0, [lumaq+lstrideq*0+16] |
| vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 |
| mova xm1, [srcq] |
| vinserti128 m1, [srcq+strideq], 1 |
| pmaddubsw m3, m2 |
| pmaddubsw m0, m2 |
| pavgw m3, m7 |
| pavgw m0, m7 |
| %else |
| mova m2, [lumaq] |
| mova m1, [srcq] |
| %endif |
| %if %1 |
| %if %2 |
| packuswb m2, m3, m0 ; luma |
| %endif |
| punpckhbw m3, m2, m1 |
| punpcklbw m2, m1 ; { luma, chroma } |
| pmaddubsw m3, m14 |
| pmaddubsw m2, m14 |
| psraw m3, 6 |
| psraw m2, 6 |
| paddw m3, m15 |
| paddw m2, m15 |
| packuswb m2, m3 ; pack+unpack = clip |
| %endif |
| %if %1 || %2 == 0 |
| punpcklbw m3, m2, m7 |
| punpckhbw m0, m2, m7 |
| %endif |
| |
| ; scaling[luma_src] |
| pandn m4, m8, m3 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m0 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| psrld m0, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m0-2], m6 |
| pblendw m2, m4, 0xaa |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %2 |
| movu xm5, [grain_lutq+offxyq+ 0] |
| vinserti128 m5, [grain_lutq+offxyq+82], 1 |
| %else |
| movu m5, [grain_lutq+offxyq] |
| %endif |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; unpack chroma_source |
| punpcklbw m0, m1, m7 |
| punpckhbw m1, m7 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| %if %2 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| %else |
| mova [dstq], m0 |
| %endif |
| |
| %if %2 |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| %else |
| add srcq, strideq |
| add dstq, strideq |
| add lumaq, lstrideq |
| %endif |
| add grain_lutq, 82<<%2 |
| sub hb, 1+%2 |
| jg %%loop_y |
| |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*(1+%2)] |
| add srcq, wq |
| add dstq, wq |
| test overlapd, overlapd |
| jz %%loop_x |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne %%loop_x_hv_overlap |
| |
| ; horizontal overlap (without vertical overlap) |
| %%loop_x_h_overlap: |
| rorx r6, seeq, 1 |
| or seed, 0xEFF4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, unused1, unused2, lstride |
| |
| lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164>>%3 |
| lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, unused1, unused2, lstride |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| %%loop_y_h_overlap: |
| ; src |
| %if %2 |
| mova xm3, [lumaq+lstrideq*0+ 0] |
| vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 |
| vpbroadcastd m2, [pb_1] |
| mova xm0, [lumaq+lstrideq*0+16] |
| vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 |
| mova xm1, [srcq] |
| vinserti128 m1, [srcq+strideq], 1 |
| pmaddubsw m3, m2 |
| pmaddubsw m0, m2 |
| pavgw m3, m7 |
| pavgw m0, m7 |
| %else |
| mova m2, [lumaq] |
| mova m1, [srcq] |
| %endif |
| %if %1 |
| %if %2 |
| packuswb m2, m3, m0 ; luma |
| %endif |
| punpckhbw m3, m2, m1 |
| punpcklbw m2, m1 ; { luma, chroma } |
| pmaddubsw m3, m14 |
| pmaddubsw m2, m14 |
| psraw m3, 6 |
| psraw m2, 6 |
| paddw m3, m15 |
| paddw m2, m15 |
| packuswb m2, m3 ; pack+unpack = clip |
| %endif |
| %if %1 || %2 == 0 |
| punpcklbw m3, m2, m7 |
| punpckhbw m0, m2, m7 |
| %endif |
| |
| ; scaling[luma_src] |
| pandn m4, m8, m3 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m0 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| psrld m0, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m0-2], m6 |
| pblendw m2, m4, 0xaa |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %2 |
| movu xm5, [grain_lutq+offxyq+ 0] |
| vinserti128 m5, [grain_lutq+offxyq+82], 1 |
| movd xm4, [grain_lutq+left_offxyq+ 0] |
| vinserti128 m4, [grain_lutq+left_offxyq+82], 1 |
| punpcklbw m4, m5 |
| %if %1 |
| vpbroadcastq m0, [pb_23_22] |
| pmaddubsw m4, m0, m4 |
| %else |
| pmaddubsw m4, m15, m4 |
| %endif |
| pmulhrsw m4, m12 |
| packsswb m4, m4 |
| vpblendd m4, m5, 0xee |
| %else |
| movu m5, [grain_lutq+offxyq] |
| movd xm4, [grain_lutq+left_offxyq] |
| punpcklbw xm4, xm5 |
| %if %1 |
| movq xm0, [pb_27_17_17_27] |
| pmaddubsw xm4, xm0, xm4 |
| %else |
| pmaddubsw xm4, xm15, xm4 |
| %endif |
| pmulhrsw xm4, xm12 |
| packsswb xm4, xm4 |
| vpblendd m4, m5, 0xfe |
| %endif |
| punpckhbw m5, m7 |
| punpcklbw m4, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; unpack chroma_source |
| punpcklbw m0, m1, m7 |
| punpckhbw m1, m7 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| %if %2 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| %else |
| mova [dstq], m0 |
| %endif |
| |
| %if %2 |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| %else |
| add srcq, strideq |
| add dstq, strideq |
| add lumaq, lstrideq |
| %endif |
| add grain_lutq, 82*(1+%2) |
| sub hb, 1+%2 |
| jg %%loop_y_h_overlap |
| |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*(1+%2)] |
| add srcq, wq |
| add dstq, wq |
| |
| ; r8m = sbym |
| cmp dword r8m, 0 |
| jne %%loop_x_hv_overlap |
| jmp %%loop_x_h_overlap |
| |
| %%vertical_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ |
| sby, see, overlap, unused1, unused2, lstride |
| |
| movzx sbyd, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, sbyd, 173 * 0x00010001 |
| imul sbyd, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add sbyd, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and sbyd, 0xff00ff00 |
| xor seed, r7d |
| xor seed, sbyd ; (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| unused1, unused2, see, overlap, unused3, unused4, lstride |
| |
| mov lumaq, r9mp |
| lea r12, [srcq+wq] |
| lea r13, [dstq+wq] |
| lea r14, [lumaq+wq*(1+%2)] |
| mov r11mp, r12 |
| mov r12mp, r13 |
| mov lstrideq, r10mp |
| neg wq |
| |
| %%loop_x_v_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, overlap, top_offxy, unused, lstride |
| |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164>>%3 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, overlap, top_offxy, unused, lstride |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| %if %2 == 0 |
| vpbroadcastd m13, [pb_27_17] |
| %endif |
| %%loop_y_v_overlap: |
| ; src |
| %if %2 |
| mova xm3, [lumaq+lstrideq*0+ 0] |
| vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 |
| vpbroadcastd m2, [pb_1] |
| mova xm0, [lumaq+lstrideq*0+16] |
| vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 |
| mova xm1, [srcq] |
| vinserti128 m1, [srcq+strideq], 1 |
| pmaddubsw m3, m2 |
| pmaddubsw m0, m2 |
| pavgw m3, m7 |
| pavgw m0, m7 |
| %else |
| mova m2, [lumaq] |
| mova m1, [srcq] |
| %endif |
| %if %1 |
| %if %2 |
| packuswb m2, m3, m0 ; luma |
| %endif |
| punpckhbw m3, m2, m1 |
| punpcklbw m2, m1 ; { luma, chroma } |
| pmaddubsw m3, m14 |
| pmaddubsw m2, m14 |
| psraw m3, 6 |
| psraw m2, 6 |
| paddw m3, m15 |
| paddw m2, m15 |
| packuswb m2, m3 ; pack+unpack = clip |
| %endif |
| %if %1 || %2 == 0 |
| punpcklbw m3, m2, m7 |
| punpckhbw m0, m2, m7 |
| %endif |
| |
| ; scaling[luma_src] |
| pandn m4, m8, m3 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m0 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| psrld m0, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m0-2], m6 |
| pblendw m2, m4, 0xaa |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %3 == 0 |
| %if %2 |
| movu xm0, [grain_lutq+offxyq] |
| vinserti128 m0, [grain_lutq+offxyq+82], 1 |
| movu xm4, [grain_lutq+top_offxyq] |
| vinserti128 m4, [grain_lutq+top_offxyq+82], 1 |
| %else |
| movu m0, [grain_lutq+offxyq] |
| movu m4, [grain_lutq+top_offxyq] |
| %endif |
| punpcklbw m5, m4, m0 |
| punpckhbw m4, m0 |
| pmaddubsw m5, m13, m5 |
| pmaddubsw m4, m13, m4 |
| pmulhrsw m5, m12 |
| pmulhrsw m4, m12 |
| packsswb m5, m4 |
| %else |
| movq xm4, [grain_lutq+offxyq] |
| vinserti128 m4, [grain_lutq+offxyq+8], 1 |
| movq xm5, [grain_lutq+top_offxyq] |
| vinserti128 m5, [grain_lutq+top_offxyq+8], 1 |
| punpcklbw m5, m4 |
| pmaddubsw m5, m13, m5 |
| pmulhrsw m5, m12 |
| vextracti128 xm4, m5, 1 |
| packsswb xm5, xm4 |
| ; only interpolate first line, insert second line unmodified |
| vinserti128 m5, [grain_lutq+offxyq+82], 1 |
| %endif |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[luma_src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; unpack chroma_source |
| punpcklbw m0, m1, m7 |
| punpckhbw m1, m7 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| %if %2 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| %else |
| mova [dstq], m0 |
| %endif |
| |
| sub hb, 1+%2 |
| jle %%end_y_v_overlap |
| %if %2 |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| %else |
| add srcq, strideq |
| add dstq, strideq |
| add lumaq, lstrideq |
| %endif |
| add grain_lutq, 82<<%2 |
| %if %2 == 0 |
| vpbroadcastd m13, [pb_17_27] |
| add hd, 0x80000000 |
| jnc %%loop_y_v_overlap |
| %endif |
| jmp %%loop_y |
| |
| %%end_y_v_overlap: |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*(1+%2)] |
| add srcq, wq |
| add dstq, wq |
| |
| ; since fg_dataq.overlap is guaranteed to be set, we never jump |
| ; back to .loop_x_v_overlap, and instead always fall-through to |
| ; h+v overlap |
| |
| %%loop_x_hv_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride |
| |
| lea topleft_offxyd, [top_offxyq+(32>>%2)] |
| lea left_offxyd, [offyq+(32>>%2)] |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164>>%3 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ |
| h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| %if %2 == 0 |
| vpbroadcastd m13, [pb_27_17] |
| %endif |
| %%loop_y_hv_overlap: |
| ; src |
| %if %2 |
| mova xm3, [lumaq+lstrideq*0+ 0] |
| vinserti128 m3, [lumaq+lstrideq*(1+%3)+ 0], 1 |
| vpbroadcastd m2, [pb_1] |
| mova xm0, [lumaq+lstrideq*0+16] |
| vinserti128 m0, [lumaq+lstrideq*(1+%3)+16], 1 |
| mova xm1, [srcq] |
| vinserti128 m1, [srcq+strideq], 1 |
| pmaddubsw m3, m2 |
| pmaddubsw m0, m2 |
| pavgw m3, m7 |
| pavgw m0, m7 |
| %else |
| mova m2, [lumaq] |
| mova m1, [srcq] |
| %endif |
| %if %1 |
| %if %2 |
| packuswb m2, m3, m0 ; luma |
| %endif |
| punpckhbw m3, m2, m1 |
| punpcklbw m2, m1 ; { luma, chroma } |
| pmaddubsw m3, m14 |
| pmaddubsw m2, m14 |
| psraw m3, 6 |
| psraw m2, 6 |
| paddw m3, m15 |
| paddw m2, m15 |
| packuswb m2, m3 ; pack+unpack = clip |
| %endif |
| %if %1 || %2 == 0 |
| punpcklbw m3, m2, m7 |
| punpckhbw m0, m2, m7 |
| %endif |
| |
| ; scaling[luma_src] |
| pandn m4, m8, m3 |
| mova m6, m8 |
| vpgatherdd m2, [scalingq+m4-0], m8 |
| psrld m3, 16 |
| mova m8, m6 |
| vpgatherdd m4, [scalingq+m3-2], m6 |
| pandn m5, m8, m0 |
| mova m6, m8 |
| vpgatherdd m3, [scalingq+m5-0], m8 |
| psrld m0, 16 |
| mova m8, m6 |
| vpgatherdd m5, [scalingq+m0-2], m6 |
| pblendw m2, m4, 0xaa |
| pblendw m3, m5, 0xaa |
| |
| ; grain = grain_lut[offy+y][offx+x] |
| %if %2 |
| movu xm4, [grain_lutq+offxyq] |
| vinserti128 m4, [grain_lutq+offxyq+82], 1 |
| movd xm0, [grain_lutq+left_offxyq] |
| vinserti128 m0, [grain_lutq+left_offxyq+82], 1 |
| movd xm6, [grain_lutq+topleft_offxyq] |
| %if %3 |
| movq xm5, [grain_lutq+top_offxyq] |
| vinserti128 m5, [grain_lutq+top_offxyq+8], 1 |
| %else |
| vinserti128 m6, [grain_lutq+topleft_offxyq+82], 1 |
| movu xm5, [grain_lutq+top_offxyq] |
| vinserti128 m5, [grain_lutq+top_offxyq+82], 1 |
| %endif |
| |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw m0, m4 |
| %if %3 |
| punpcklbw xm6, xm5 |
| %else |
| punpcklbw m6, m5 |
| %endif |
| punpcklqdq m0, m6 |
| %if %1 |
| vpbroadcastq m6, [pb_23_22] |
| pmaddubsw m0, m6, m0 |
| %else |
| pmaddubsw m0, m15, m0 |
| %endif |
| pmulhrsw m0, m12 |
| packsswb m0, m0 |
| vpblendd m4, m0, 0x11 |
| %if %3 |
| pshuflw xm0, xm0, q1032 |
| vpblendd m5, m0, 0x01 |
| %else |
| pshuflw m0, m0, q1032 |
| vpblendd m5, m0, 0x11 |
| %endif |
| %else |
| movu m4, [grain_lutq+offxyq] |
| movd xm0, [grain_lutq+left_offxyq] |
| movu m5, [grain_lutq+top_offxyq] |
| movd xm6, [grain_lutq+topleft_offxyq] |
| punpcklbw xm0, xm4 |
| punpcklbw xm6, xm5 |
| punpcklqdq xm0, xm6 |
| %if %1 |
| vpbroadcastq xm6, [pb_27_17_17_27] |
| pmaddubsw xm0, xm6, xm0 |
| %else |
| pmaddubsw xm0, xm15, xm0 |
| %endif |
| pmulhrsw xm0, xm12 |
| packsswb xm0, xm0 |
| vpblendd m4, m0, 0x01 |
| pshuflw xm0, xm0, q1032 |
| vpblendd m5, m0, 0x01 |
| %endif |
| |
| ; followed by v interpolation (top | cur -> cur) |
| %if %3 |
| vpermq m0, m4, q3120 |
| punpcklbw m5, m0 |
| pmaddubsw m5, m13, m5 |
| pmulhrsw m5, m12 |
| vextracti128 xm0, m5, 1 |
| packsswb xm5, xm0 |
| vpblendd m5, m4, 0xf0 |
| %else |
| punpckhbw m0, m5, m4 |
| punpcklbw m5, m4 |
| pmaddubsw m4, m13, m0 |
| pmaddubsw m5, m13, m5 |
| pmulhrsw m4, m12 |
| pmulhrsw m5, m12 |
| packsswb m5, m4 |
| %endif |
| punpcklbw m4, m5, m7 |
| punpckhbw m5, m7 |
| |
| ; noise = round2(scaling[src] * grain, scaling_shift) |
| pmaddubsw m2, m4 |
| pmaddubsw m3, m5 |
| pmulhrsw m2, m9 |
| pmulhrsw m3, m9 |
| |
| ; unpack chroma source |
| punpcklbw m0, m1, m7 |
| punpckhbw m1, m7 |
| |
| ; dst = clip_pixel(src, noise) |
| paddw m0, m2 |
| paddw m1, m3 |
| packuswb m0, m1 |
| pmaxub m0, m10 |
| pminub m0, m11 |
| %if %2 |
| mova [dstq], xm0 |
| vextracti128 [dstq+strideq], m0, 1 |
| %else |
| mova [dstq], m0 |
| %endif |
| |
| %if %2 |
| lea srcq, [srcq+strideq*2] |
| lea dstq, [dstq+strideq*2] |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| %else |
| add srcq, strideq |
| add dstq, strideq |
| add lumaq, lstrideq |
| %endif |
| add grain_lutq, 82<<%2 |
| sub hb, 1+%2 |
| %if %2 |
| jg %%loop_y_h_overlap |
| %else |
| je %%end_y_hv_overlap |
| vpbroadcastd m13, [pb_17_27] |
| add hd, 0x80000000 |
| jnc %%loop_y_hv_overlap |
| jmp %%loop_y_h_overlap |
| %endif |
| |
| %%end_y_hv_overlap: |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r14+wq*(1+%2)] |
| add srcq, wq |
| add dstq, wq |
| jmp %%loop_x_hv_overlap |
| %endmacro |
| |
| %%FGUV_32x32xN_LOOP 1, %2, %3 |
| .csfl: |
| %%FGUV_32x32xN_LOOP 0, %2, %3 |
| .end: |
| RET |
| %endmacro |
| |
| GEN_GRAIN_UV_FN 420, 1, 1 |
| FGUV_FN 420, 1, 1 |
| GEN_GRAIN_UV_FN 422, 1, 0 |
| FGUV_FN 422, 1, 0 |
| GEN_GRAIN_UV_FN 444, 0, 0 |
| FGUV_FN 444, 0, 0 |
| |
| %endif ; ARCH_X86_64 |
