| ; Copyright © 2022, VideoLAN and dav1d authors |
| ; Copyright © 2022, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "config.asm" |
| %include "ext/x86/x86inc.asm" |
| %include "x86/filmgrain_common.asm" |
| |
| %if ARCH_X86_64 |
| |
| SECTION_RODATA 64 |
| |
| pb_even: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 |
| db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 |
| db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94 |
| db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126 |
| pb_odd: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 |
| db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 |
| db 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95 |
| db 97, 99,101,103,105,107,109,111,113,115,117,119,121,123,125,127 |
| interleave_hl: db 8, 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7 |
| pb_27_17_17_27: db 27, 17, 17, 27, 0, 32, 0, 32 |
| pb_23_22_0_32: db 23, 22, 0, 32, 0, 32, 0, 32 |
| pb_27_17: times 2 db 27, 17 |
| pb_23_22: times 2 db 23, 22 |
| pw_8: times 2 dw 8 |
| pw_1024: times 2 dw 1024 |
| pb_17_27: times 2 db 17, 27 |
| fg_max: times 4 db 255 |
| times 4 db 240 |
| times 4 db 235 |
| fg_min: times 4 db 0 |
| times 4 db 16 |
| noise_rnd: times 2 dw 128 |
| times 2 dw 64 |
| times 2 dw 32 |
| times 2 dw 16 |
| |
| SECTION .text |
| |
| INIT_ZMM avx512icl |
| cglobal fgy_32x32xn_8bpc, 6, 13, 22, dst, src, stride, fg_data, w, scaling, \ |
| grain_lut, h, sby, see, overlap |
| %define base r11-fg_min |
| lea r11, [fg_min] |
| mov r6d, [fg_dataq+FGData.scaling_shift] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| mov sbyd, sbym |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| mov r12, 0x0000000f0000000f ; h_overlap mask |
| mova m0, [scalingq+64*0] |
| mova m1, [scalingq+64*1] |
| mova m2, [scalingq+64*2] |
| mova m3, [scalingq+64*3] |
| kmovq k1, r12 |
| vbroadcasti32x4 m4, [base+interleave_hl] |
| vpbroadcastd ym16, [base+pb_27_17] |
| vpbroadcastd m12, [base+pb_17_27] |
| vpbroadcastd m6, [base+noise_rnd+r6*4-32] |
| test sbyd, sbyd |
| setnz r6b |
| vpbroadcastd m7, [base+fg_min+r7*4] |
| vpbroadcastd m8, [base+fg_max+r7*8] |
| pxor m5, m5 |
| vpbroadcastd m9, [base+pw_1024] |
| vpbroadcastq m10, [base+pb_27_17_17_27] |
| vmovdqa64 m12{k1}, m16 |
| test r6b, overlapb |
| jnz .v_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rorx seed, seed, 24 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| h, sby, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| .loop_x: |
| rorx r6, seeq, 1 |
| or seed, 0xeff4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offxd, [offyq+offxq*2+829] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| h, sby, see, overlap |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| .loop_y: |
| movu ym21, [grain_lutq+offxyq-82] |
| vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1 |
| call .add_noise |
| sub hb, 2 |
| jg .loop_y |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| test overlapd, overlapd |
| jz .loop_x |
| test sbyd, sbyd |
| jnz .hv_overlap |
| |
| .loop_x_h_overlap: |
| rorx r6, seeq, 1 |
| or seed, 0xeff4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| h, sby, see, left_offxy |
| |
| rorx offyd, seed, 8 |
| mov left_offxyd, offxd ; previous column's offy*stride |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164 |
| lea offxd, [offyq+offxq*2+829] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| h, sby, see, left_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| .loop_y_h_overlap: |
| movu ym20, [grain_lutq+offxyq-82] |
| vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1 |
| movd xm19, [grain_lutq+left_offxyq-50] |
| vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2 |
| punpcklbw m19, m20 |
| pmaddubsw m19, m10, m19 |
| pmulhrsw m19, m9 |
| punpckhbw m21, m20, m5 |
| packsswb m20{k1}, m19, m19 |
| punpcklbw m20, m5, m20 |
| call .add_noise_h |
| sub hb, 2 |
| jg .loop_y_h_overlap |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| test sbyd, sbyd |
| jnz .hv_overlap |
| jmp .loop_x_h_overlap |
| |
| .v_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, offy, offx, \ |
| h, sby, see, overlap |
| |
| movzx r6d, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, r6d, 173 * 0x00010001 |
| imul r6d, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add r6d, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and r6d, 0xff00ff00 |
| xor seed, r7d |
| xor seed, r6d ; (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| h, sby, see, overlap |
| |
| lea src_bakq, [srcq+wq] |
| neg wq |
| sub dstq, srcq |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offxd, [offyq+offxq*2+0x10001*829+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| h, sby, see, overlap, top_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| movu ym19, [grain_lutq+offxyq-82] |
| vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 |
| movu ym21, [grain_lutq+top_offxyq-82] |
| vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 |
| punpckhbw m20, m21, m19 |
| punpcklbw m21, m19 |
| call .add_noise_v |
| sub hb, 2 |
| jg .loop_y |
| add wq, 32 |
| jge .end |
| lea srcq, [src_bakq+wq] |
| |
| ; since fg_dataq.overlap is guaranteed to be set, we never jump back |
| ; to .v_overlap, and instead always fall-through to h+v overlap |
| .hv_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offx, offy, \ |
| h, sby, see, left_offxy, top_offxy, topleft_offxy |
| |
| mov topleft_offxyd, top_offxyd |
| rorx offyd, seed, 8 |
| mov left_offxyd, offxd |
| rorx offxd, seed, 12 |
| and offyd, 0xf000f |
| and offxd, 0xf000f |
| imul offyd, 164 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offxd, [offyq+offxq*2+0x10001*829+32*82] |
| |
| DEFINE_ARGS dst, src, stride, src_bak, w, offxy, grain_lut, \ |
| h, sby, see, left_offxy, top_offxy, topleft_offxy |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| movu ym19, [grain_lutq+offxyq-82] |
| vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 |
| movd xm16, [grain_lutq+left_offxyq-50] |
| vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2 |
| movu ym21, [grain_lutq+top_offxyq-82] |
| vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 |
| movd xm17, [grain_lutq+topleft_offxyq-50] |
| vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2 |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw m16, m19 |
| pmaddubsw m16, m10, m16 |
| punpcklbw m17, m21 |
| pmaddubsw m17, m10, m17 |
| punpckhbw m20, m21, m19 |
| pmulhrsw m16, m9 |
| pmulhrsw m17, m9 |
| packsswb m19{k1}, m16, m16 |
| packsswb m21{k1}, m17, m17 |
| ; followed by v interpolation (top | cur -> cur) |
| punpcklbw m21, m19 |
| call .add_noise_v |
| sub hb, 2 |
| jg .loop_y_h_overlap |
| add wq, 32 |
| lea srcq, [src_bakq+wq] |
| jl .hv_overlap |
| .end: |
| RET |
| ALIGN function_align |
| .add_noise_v: |
| pmaddubsw m20, m12, m20 |
| pmaddubsw m21, m12, m21 |
| pmulhrsw m20, m9 |
| pmulhrsw m21, m9 |
| packsswb m21, m20 |
| .add_noise: |
| punpcklbw m20, m5, m21 |
| punpckhbw m21, m5 |
| .add_noise_h: |
| mova ym18, [srcq+strideq*0] |
| vinserti32x8 m18, [srcq+strideq*1], 1 |
| mova m19, m0 |
| punpcklbw m16, m18, m5 |
| vpermt2b m19, m18, m1 ; scaling[ 0..127] |
| vpmovb2m k2, m18 |
| punpckhbw m17, m18, m5 |
| vpermi2b m18, m2, m3 ; scaling[128..255] |
| vmovdqu8 m19{k2}, m18 ; scaling[src] |
| pshufb m19, m4 |
| pmaddubsw m18, m19, m20 |
| pmaddubsw m19, m21 |
| add grain_lutq, 82*2 |
| pmulhrsw m18, m6 ; noise |
| pmulhrsw m19, m6 |
| paddw m16, m18 |
| paddw m17, m19 |
| packuswb m16, m17 |
| pmaxub m16, m7 |
| pminub m16, m8 |
| mova [dstq+srcq], ym16 |
| add srcq, strideq |
| vextracti32x8 [dstq+srcq], m16, 1 |
| add srcq, strideq |
| ret |
| |
| %macro FGUV_FN 3 ; name, ss_hor, ss_ver |
| cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \ |
| scaling, grain_lut, h, sby, luma, \ |
| overlap, uv_pl, is_id, _, stride3 |
| lea r11, [fg_min] |
| mov r6d, [fg_dataq+FGData.scaling_shift] |
| mov r7d, [fg_dataq+FGData.clip_to_restricted_range] |
| mov r9d, is_idm |
| mov sbyd, sbym |
| mov overlapd, [fg_dataq+FGData.overlap_flag] |
| %if %2 |
| mov r12, 0x000f000f000f000f ; h_overlap mask |
| vpbroadcastq m10, [base+pb_23_22_0_32] |
| lea stride3q, [strideq*3] |
| %else |
| mov r12, 0x0000000f0000000f |
| vpbroadcastq m10, [base+pb_27_17_17_27] |
| %endif |
| mova m0, [scalingq+64*0] |
| mova m1, [scalingq+64*1] |
| mova m2, [scalingq+64*2] |
| mova m3, [scalingq+64*3] |
| kmovq k1, r12 |
| vbroadcasti32x4 m4, [base+interleave_hl] |
| vpbroadcastd m6, [base+noise_rnd+r6*4-32] |
| vpbroadcastd m7, [base+fg_min+r7*4] |
| shlx r7d, r7d, r9d |
| vpbroadcastd m8, [base+fg_max+r7*4] |
| test sbyd, sbyd |
| setnz r7b |
| vpbroadcastd m9, [base+pw_1024] |
| mova m11, [base+pb_even] |
| mova m12, [base+pb_odd] |
| pxor m5, m5 |
| mov r5, r10mp ; lstride |
| cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 |
| jne .csfl |
| |
| %macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver |
| DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ |
| h, sby, see, overlap, uv_pl, _, _, stride3 |
| %if %1 |
| mov r6d, uv_plm |
| vpbroadcastd m16, [base+pw_8] |
| vbroadcasti32x4 m14, [fg_dataq+FGData.uv_mult+r6*4] |
| vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r6*4] |
| pshufb m14, m16 ; uv_luma_mult, uv_mult |
| %endif |
| test r7b, overlapb |
| jnz %%v_overlap |
| |
| imul seed, sbyd, (173 << 24) | 37 |
| add seed, (105 << 24) | 178 |
| rorx seed, seed, 24 |
| movzx seed, seew |
| xor seed, [fg_dataq+FGData.seed] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| offx, offy, see, overlap, _, _, _, stride3 |
| |
| mov lumaq, r9mp |
| lea r11, [srcq+wq] |
| lea r12, [dstq+wq] |
| lea r13, [lumaq+wq*(1+%2)] |
| mov r11mp, r11 |
| mov r12mp, r12 |
| neg wq |
| |
| %%loop_x: |
| rorx r6, seeq, 1 |
| or seed, 0xeff4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164>>%3 |
| lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| h, offxy, see, overlap, _, _, _, stride3 |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| %%loop_y: |
| %if %2 |
| movu xm21, [grain_lutq+offxyq+82*0] |
| vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 |
| vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| %else |
| movu ym21, [grain_lutq+offxyq+82*0] |
| vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| %endif |
| call %%add_noise |
| sub hb, 2<<%2 |
| jg %%loop_y |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r13+wq*(1<<%2)] |
| add srcq, wq |
| add dstq, wq |
| test overlapd, overlapd |
| jz %%loop_x |
| cmp dword r8m, 0 ; sby |
| jne %%hv_overlap |
| |
| ; horizontal overlap (without vertical overlap) |
| %%loop_x_h_overlap: |
| rorx r6, seeq, 1 |
| or seed, 0xeff4 |
| test seeb, seeh |
| lea seed, [r6+0x8000] |
| cmovp seed, r6d ; updated seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| offx, offy, see, left_offxy, _, _, _, stride3 |
| |
| lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx |
| rorx offyd, seed, 8 |
| rorx offxq, seeq, 12 |
| and offyd, 0xf |
| imul offyd, 164>>%3 |
| lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| h, offxy, see, left_offxy, _, _, _, stride3 |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| %%loop_y_h_overlap: |
| %if %2 |
| movu xm20, [grain_lutq+offxyq +82*0] |
| movd xm19, [grain_lutq+left_offxyq+82*0] |
| vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1 |
| vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1 |
| vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2 |
| vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2 |
| vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3 |
| vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3 |
| %else |
| movu ym20, [grain_lutq+offxyq + 0] |
| movd xm19, [grain_lutq+left_offxyq+ 0] |
| vinserti32x8 m20, [grain_lutq+offxyq +82], 1 |
| vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2 |
| %endif |
| punpcklbw m19, m20 |
| pmaddubsw m19, m10, m19 |
| punpckhbw m21, m20, m5 |
| pmulhrsw m19, m9 |
| vpacksswb m20{k1}, m19, m19 |
| punpcklbw m20, m5, m20 |
| call %%add_noise_h |
| sub hb, 2<<%2 |
| jg %%loop_y_h_overlap |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r13+wq*(1<<%2)] |
| add srcq, wq |
| add dstq, wq |
| cmp dword r8m, 0 ; sby |
| jne %%hv_overlap |
| jmp %%loop_x_h_overlap |
| |
| %%v_overlap: |
| DEFINE_ARGS dst, src, stride, fg_data, w, lstride, grain_lut, \ |
| _, sby, see, overlap, _, _, _, stride3 |
| |
| movzx sbyd, sbyb |
| imul seed, [fg_dataq+FGData.seed], 0x00010001 |
| imul r7d, sbyd, 173 * 0x00010001 |
| imul sbyd, 37 * 0x01000100 |
| add r7d, (105 << 16) | 188 |
| add sbyd, (178 << 24) | (141 << 8) |
| and r7d, 0x00ff00ff |
| and sbyd, 0xff00ff00 |
| xor seed, r7d |
| xor seed, sbyd ; (cur_seed << 16) | top_seed |
| |
| %if %3 |
| vpbroadcastd m13, [base+pb_23_22] |
| kxnorw k3, k3, k3 ; v_overlap mask |
| %elif %2 |
| vbroadcasti32x8 m13, [base+pb_27_17] |
| kxnord k3, k3, k3 |
| pshufd m13, m13, q0000 ; 8x27_17, 8x17_27 |
| %else |
| vpbroadcastd ym16, [base+pb_27_17] |
| vpbroadcastd m13, [base+pb_17_27] |
| vmovdqa64 m13{k1}, m16 |
| %endif |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| offx, offy, see, overlap, top_offxy, _, _, stride3 |
| |
| mov lumaq, r9mp |
| lea r11, [srcq+wq] |
| lea r12, [dstq+wq] |
| lea r13, [lumaq+wq*(1<<%2)] |
| mov r11mp, r11 |
| mov r12mp, r12 |
| neg wq |
| |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0x000f000f |
| and offxd, 0x000f000f |
| imul offyd, 164>>%3 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| h, offxy, see, overlap, top_offxy, _, _, stride3 |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| %if %3 |
| movu xm18, [grain_lutq+offxyq+82*0] |
| movu xm20, [grain_lutq+top_offxyq+82*0] |
| ; only interpolate first line, insert remaining line unmodified |
| vbroadcasti128 ym21, [grain_lutq+offxyq+82*1] |
| vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| punpcklbw xm19, xm20, xm18 |
| punpckhbw xm20, xm18 |
| %elif %2 |
| movu xm18, [grain_lutq+offxyq+82*0] |
| vinserti128 ym18, [grain_lutq+offxyq+82*1], 1 |
| movu xm20, [grain_lutq+top_offxyq+82*0] |
| vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1 |
| vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2] |
| vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| punpcklbw ym19, ym20, ym18 |
| punpckhbw ym20, ym18 |
| %else |
| movu ym21, [grain_lutq+offxyq+82*0] |
| vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| movu ym20, [grain_lutq+top_offxyq+82*0] |
| vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 |
| %endif |
| call %%add_noise_v |
| sub hb, 2<<%2 |
| jg %%loop_y |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r13+wq*(1<<%2)] |
| add srcq, wq |
| add dstq, wq |
| |
| %%hv_overlap: |
| ; we assume from the block above that bits 8-15 of r7d are zero'ed |
| mov r6d, seed |
| or seed, 0xeff4eff4 |
| test seeb, seeh |
| setp r7b ; parity of top_seed |
| shr seed, 16 |
| shl r7d, 16 |
| test seeb, seeh |
| setp r7b ; parity of cur_seed |
| or r6d, 0x00010001 |
| xor r7d, r6d |
| rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| offx, offy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 |
| |
| lea topleft_offxyd, [top_offxyq+(32>>%2)] |
| lea left_offxyd, [offyq+(32>>%2)] |
| rorx offyd, seed, 8 |
| rorx offxd, seed, 12 |
| and offyd, 0x000f000f |
| and offxd, 0x000f000f |
| imul offyd, 164>>%3 |
| ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy |
| lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] |
| |
| DEFINE_ARGS dst, src, stride, luma, w, lstride, grain_lut, \ |
| h, offxy, see, left_offxy, top_offxy, topleft_offxy, _, stride3 |
| |
| mov grain_lutq, grain_lutmp |
| mov hd, hm |
| movzx top_offxyd, offxyw |
| shr offxyd, 16 |
| |
| %if %2 |
| movu xm21, [grain_lutq+offxyq+82*0] |
| movd xm16, [grain_lutq+left_offxyq+82*0] |
| vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 |
| vinserti128 ym16, [grain_lutq+left_offxyq+82*1], 1 |
| vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 |
| vinserti32x4 m16, [grain_lutq+left_offxyq+82*2], 2 |
| vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 |
| vinserti32x4 m16, [grain_lutq+left_offxyq+82*3], 3 |
| movd xm18, [grain_lutq+topleft_offxyq+82*0] |
| movu xm20, [grain_lutq+top_offxyq] |
| ; do h interpolation first (so top | top/left -> top, left | cur -> cur) |
| punpcklbw m16, m21 |
| %if %3 |
| punpcklbw xm18, xm20 |
| %else |
| vinserti128 ym18, [grain_lutq+topleft_offxyq+82*1], 1 |
| vinserti128 ym20, [grain_lutq+top_offxyq+82*1], 1 |
| punpcklbw ym18, ym20 |
| %endif |
| punpcklqdq m16, m18 |
| pmaddubsw m16, m10, m16 |
| pmulhrsw m16, m9 |
| packsswb m16, m16 |
| vmovdqu8 m21{k1}, m16 |
| %if %3 |
| vpalignr xm20{k1}, xm16, xm16, 4 |
| punpcklbw xm19, xm20, xm21 |
| punpckhbw xm20, xm21 |
| %else |
| vpalignr ym20{k1}, ym16, ym16, 4 |
| punpcklbw ym19, ym20, ym21 |
| punpckhbw ym20, ym21 |
| %endif |
| %else |
| movu ym21, [grain_lutq+offxyq+82*0] |
| vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 |
| movd xm16, [grain_lutq+left_offxyq+82*0] |
| vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2 |
| movu ym20, [grain_lutq+top_offxyq+82*0] |
| vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 |
| movd xm18, [grain_lutq+topleft_offxyq+82*0] |
| vinserti32x4 m18, [grain_lutq+topleft_offxyq+82*1], 2 |
| punpcklbw m16, m21 |
| punpcklbw m18, m20 |
| punpcklqdq m16, m18 |
| pmaddubsw m16, m10, m16 |
| pmulhrsw m16, m9 |
| packsswb m16, m16 |
| vpalignr m20{k1}, m16, m16, 4 |
| vmovdqu8 m21{k1}, m16 |
| %endif |
| call %%add_noise_v |
| sub hb, 2<<%2 |
| jg %%loop_y_h_overlap |
| add wq, 32>>%2 |
| jge .end |
| mov srcq, r11mp |
| mov dstq, r12mp |
| lea lumaq, [r13+wq*(1<<%2)] |
| add srcq, wq |
| add dstq, wq |
| jmp %%hv_overlap |
| ALIGN function_align |
| %%add_noise_v: |
| %if %3 |
| pmaddubsw xm19, xm13, xm19 |
| pmaddubsw xm20, xm13, xm20 |
| pmulhrsw xm19, xm9 |
| pmulhrsw xm20, xm9 |
| vpacksswb m21{k3}, m19, m20 |
| %elif %2 |
| pmaddubsw ym19, ym13, ym19 |
| pmaddubsw ym20, ym13, ym20 |
| pmulhrsw ym19, ym9 |
| pmulhrsw ym20, ym9 |
| vpacksswb m21{k3}, m19, m20 |
| %else |
| punpcklbw m19, m20, m21 |
| punpckhbw m20, m21 |
| pmaddubsw m19, m13, m19 |
| pmaddubsw m20, m13, m20 |
| pmulhrsw m19, m9 |
| pmulhrsw m20, m9 |
| packsswb m21, m19, m20 |
| %endif |
| %%add_noise: |
| punpcklbw m20, m5, m21 |
| punpckhbw m21, m5 |
| %%add_noise_h: |
| mova ym18, [lumaq+lstrideq*(0<<%3)] |
| vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 |
| %if %2 |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| mova ym16, [lumaq+lstrideq*(0<<%3)] |
| vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1 |
| mova xm17, [srcq+strideq*0] |
| mova m19, m11 |
| vpermi2b m19, m18, m16 |
| vinserti128 ym17, [srcq+strideq*1], 1 |
| vpermt2b m18, m12, m16 |
| vinserti32x4 m17, [srcq+strideq*2], 2 |
| pavgb m18, m19 |
| vinserti32x4 m17, [srcq+stride3q ], 3 |
| %else |
| mova ym17, [srcq+strideq*0] |
| vinserti32x8 m17, [srcq+strideq*1], 1 |
| %endif |
| %if %1 |
| punpckhbw m19, m18, m17 |
| punpcklbw m18, m17 ; { luma, chroma } |
| pmaddubsw m19, m14 |
| pmaddubsw m18, m14 |
| psraw m19, 6 |
| psraw m18, 6 |
| paddw m19, m15 |
| paddw m18, m15 |
| packuswb m18, m19 |
| .add_noise_main: |
| mova m19, m0 |
| vpermt2b m19, m18, m1 ; scaling[ 0..127] |
| vpmovb2m k2, m18 |
| vpermi2b m18, m2, m3 ; scaling[128..255] |
| vmovdqu8 m19{k2}, m18 ; scaling[src] |
| pshufb m19, m4 |
| pmaddubsw m18, m19, m20 |
| pmaddubsw m19, m21 |
| add grain_lutq, 82*2<<%2 |
| lea lumaq, [lumaq+lstrideq*(2<<%3)] |
| lea srcq, [srcq+strideq*(2<<%2)] |
| pmulhrsw m18, m6 ; noise |
| pmulhrsw m19, m6 |
| punpcklbw m16, m17, m5 ; chroma |
| punpckhbw m17, m5 |
| paddw m16, m18 |
| paddw m17, m19 |
| packuswb m16, m17 |
| pmaxub m16, m7 |
| pminub m16, m8 |
| %if %2 |
| mova [dstq+strideq*0], xm16 |
| vextracti128 [dstq+strideq*1], ym16, 1 |
| vextracti32x4 [dstq+strideq*2], m16, 2 |
| vextracti32x4 [dstq+stride3q ], m16, 3 |
| %else |
| mova [dstq+strideq*0], ym16 |
| vextracti32x8 [dstq+strideq*1], m16, 1 |
| %endif |
| lea dstq, [dstq+strideq*(2<<%2)] |
| ret |
| %else |
| jmp .add_noise_main |
| %endif |
| %endmacro |
| |
| %%FGUV_32x32xN_LOOP 1, %2, %3 |
| .csfl: |
| %%FGUV_32x32xN_LOOP 0, %2, %3 |
| .end: |
| RET |
| %endmacro |
| |
| FGUV_FN 420, 1, 1 |
| FGUV_FN 422, 1, 0 |
| FGUV_FN 444, 0, 0 |
| |
| %endif ; ARCH_X86_64 |
