| ; Copyright © 2020, VideoLAN and dav1d authors |
| ; Copyright © 2020, Two Orioles, LLC |
| ; All rights reserved. |
| ; |
| ; Redistribution and use in source and binary forms, with or without |
| ; modification, are permitted provided that the following conditions are met: |
| ; |
| ; 1. Redistributions of source code must retain the above copyright notice, this |
| ; list of conditions and the following disclaimer. |
| ; |
| ; 2. Redistributions in binary form must reproduce the above copyright notice, |
| ; this list of conditions and the following disclaimer in the documentation |
| ; and/or other materials provided with the distribution. |
| ; |
| ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| %include "config.asm" |
| %include "ext/x86/x86inc.asm" |
| |
| %if ARCH_X86_64 |
| |
| SECTION_RODATA 64 |
| |
| spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 |
| db 32, 33, 34, 35, 34, 35, 36, 37, 36, 37, 38, 39, 38, 39, 40, 41 |
| spel_h_shufC: db 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15, 16, 17 |
| db 40, 41, 42, 43, 42, 43, 44, 45, 44, 45, 46, 47, 46, 47, 48, 49 |
| db 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23, 24, 25 |
| db 48, 49, 50, 51, 50, 51, 52, 53, 52, 53, 54, 55, 54, 55, 56, 57 |
| spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 |
| db 36, 37, 38, 39, 38, 39, 40, 41, 40, 41, 42, 43, 42, 43, 44, 45 |
| spel_h_shufD: db 12, 13, 14, 15, 14, 15, 16, 17, 16, 17, 18, 19, 18, 19, 20, 21 |
| db 44, 45, 46, 47, 46, 47, 48, 49, 48, 49, 50, 51, 50, 51, 52, 53 |
| db 20, 21, 22, 23, 22, 23, 24, 25, 24, 25, 26, 27, 26, 27, 28, 29 |
| db 52, 53, 54, 55, 54, 55, 56, 57, 56, 57, 58, 59, 58, 59, 60, 61 |
| spel_v_shuf8: db 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 |
| db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39 |
| db 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 |
| db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47 |
| spel_v_shuf16: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 |
| db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 |
| db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 |
| db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 |
| prep_endA: db 1, 2, 5, 6, 9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30 |
| db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62 |
| db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94 |
| db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126 |
| prep_endB: db 1, 2, 5, 6, 9, 10, 13, 14, 33, 34, 37, 38, 41, 42, 45, 46 |
| db 17, 18, 21, 22, 25, 26, 29, 30, 49, 50, 53, 54, 57, 58, 61, 62 |
| db 65, 66, 69, 70, 73, 74, 77, 78, 97, 98,101,102,105,106,109,110 |
| db 81, 82, 85, 86, 89, 90, 93, 94,113,114,117,118,121,122,125,126 |
| prep_endC: db 1, 2, 5, 6, 9, 10, 13, 14, 65, 66, 69, 70, 73, 74, 77, 78 |
| db 17, 18, 21, 22, 25, 26, 29, 30, 81, 82, 85, 86, 89, 90, 93, 94 |
| db 33, 34, 37, 38, 41, 42, 45, 46, 97, 98,101,102,105,106,109,110 |
| db 49, 50, 53, 54, 57, 58, 61, 62,113,114,117,118,121,122,125,126 |
| spel_shuf4a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30 |
| db 17, 18, 33, 34, 21, 22, 37, 38, 25, 26, 41, 42, 29, 30, 45, 46 |
| spel_shuf4b: db 18, 19, 33, 34, 22, 23, 37, 38, 26, 27, 41, 42, 30, 31, 45, 46 |
| db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62 |
| spel_shuf8a: db 1, 2, 17, 18, 5, 6, 21, 22, 9, 10, 25, 26, 13, 14, 29, 30 |
| db 17, 18, 65, 66, 21, 22, 69, 70, 25, 26, 73, 74, 29, 30, 77, 78 |
| db 33, 34, 49, 50, 37, 38, 53, 54, 41, 42, 57, 58, 45, 46, 61, 62 |
| db 49, 50, 97, 98, 53, 54,101,102, 57, 58,105,106, 61, 62,109,110 |
| spel_shuf8b: db 18, 19, 65, 66, 22, 23, 69, 70, 26, 27, 73, 74, 30, 31, 77, 78 |
| db 65, 66, 81, 82, 69, 70, 85, 86, 73, 74, 89, 90, 77, 78, 93, 94 |
| db 50, 51, 97, 98, 54, 55,101,102, 58, 59,105,106, 62, 63,109,110 |
| db 97, 98,113,114,101,102,117,118,105,106,121,122,109,110,125,126 |
| spel_shuf16: db 1, 2, 33, 34, 5, 6, 37, 38, 9, 10, 41, 42, 13, 14, 45, 46 |
| db 17, 18, 49, 50, 21, 22, 53, 54, 25, 26, 57, 58, 29, 30, 61, 62 |
| db 65, 66, 97, 98, 69, 70,101,102, 73, 74,105,106, 77, 78,109,110 |
| db 81, 82,113,114, 85, 86,117,118, 89, 90,121,122, 93, 94,125,126 |
| spel_shuf32: db 1, 2, 65, 66, 5, 6, 69, 70, 9, 10, 73, 74, 13, 14, 77, 78 |
| db 17, 18, 81, 82, 21, 22, 85, 86, 25, 26, 89, 90, 29, 30, 93, 94 |
| db 33, 34, 97, 98, 37, 38,101,102, 41, 42,105,106, 45, 46,109,110 |
| db 49, 50,113,114, 53, 54,117,118, 57, 58,121,122, 61, 62,125,126 |
| spel_h_shuf2b: db 1, 2, 17, 18, 5, 6, 21, 22, 17, 18, 33, 34, 21, 22, 37, 38 |
| db 33, 34, 49, 50, 37, 38, 53, 54, 49, 50, 9, 10, 53, 54, 13, 14 |
| db 9, 10, 25, 26, 13, 14, 29, 30, 25, 26, 41, 42, 29, 30, 45, 46 |
| spel_shuf2: db 10, 11, 17, 18, 14, 15, 21, 22, 17, 18, 25, 26, 21, 22, 29, 30 |
| spel_h_shuf2a: db 0, 1, 2, 3, 2, 3, 4, 5, 16, 17, 18, 19, 18, 19, 20, 21 |
| db 4, 5, 6, 7, 6, 7, 8, 9, 20, 21, 22, 23, 22, 23, 24, 25 |
| w_mask_end42x: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 |
| db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 |
| w_mask_end444: db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 |
| db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 |
| db 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94 |
| db 96, 98,100,102,104,106,108,110,112,114,116,118,120,122,124,126 |
| w_mask_shuf4: db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 |
| db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 |
| db 64, 66, 72, 74, 68, 70, 76, 78, 80, 82, 88, 90, 84, 86, 92, 94 |
| db 96, 98,104,106,100,102,108,110,112,114,120,122,116,118,124,126 |
| w_mask_shuf8: db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 |
| db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 |
| db 64, 66, 80, 82, 68, 70, 84, 86, 72, 74, 88, 90, 76, 78, 92, 94 |
| db 96, 98,112,114,100,102,116,118,104,106,120,122,108,110,124,126 |
| w_mask_shuf16: db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 |
| db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 |
| db 64, 66, 96, 98, 68, 70,100,102, 72, 74,104,106, 76, 78,108,110 |
| db 80, 82,112,114, 84, 86,116,118, 88, 90,120,122, 92, 94,124,126 |
| warp8x8_permA: db 0, 1, 2, 3, 32, 33, 34, 35, 2, 3, 4, 5, 34, 35, 36, 37 |
| db 4, 5, 6, 7, 36, 37, 38, 39, 6, 7, 8, 9, 38, 39, 40, 41 |
| db 8, 9, 10, 11, 40, 41, 42, 43, 10, 11, 12, 13, 42, 43, 44, 45 |
| db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49 |
| warp8x8_permB: db 12, 13, 14, 15, 44, 45, 46, 47, 14, 15, 16, 17, 46, 47, 48, 49 |
| db 16, 17, 18, 19, 48, 49, 50, 51, 18, 19, 20, 21, 50, 51, 52, 53 |
| db 20, 21, 22, 23, 52, 53, 54, 55, 22, 23, 24, 25, 54, 55, 56, 57 |
| db 24, 25, 26, 27, 56, 57, 58, 59, 26, 27, 28, 29, 58, 59, 60, 61 |
| warp8x8_end: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53 |
| db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55 |
| db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61 |
| db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63 |
| deint_q_shuf: ;dq 0, 2, 4, 6, 1, 3, 5, 7 |
| pd_0to7: dd 0, 1, 2, 3, 4, 5, 6, 7 |
| dd 1 |
| pw_2048: times 2 dw 2048 |
| dd 3 |
| pw_8192: times 2 dw 8192 |
| avg_shift: dw 5, 5, 3, 3 |
| pw_27615: times 2 dw 27615 |
| pw_32766: times 2 dw 32766 |
| warp8x8_permC: db -1, 0, -1, 1, -1, 8, -1, 9, -1, 4, -1, 5, -1, 12, -1, 13 |
| warp8x8_permD: db -1, 2, -1, 3, -1, 10, -1, 11, -1, 6, -1, 7, -1, 14, -1, 15 |
| warp_shift_h: db 11, 19, 11, 19, 43, 51, 43, 51, 13, 21, 13, 21, 45, 53, 45, 53 |
| blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 |
| resize_permA: dd 0, 4, 8, 12, 1, 5, 9, 13, 16, 20, 24, 28, 17, 21, 25, 29 |
| resize_permB: dd 2, 6, 10, 14, 3, 7, 11, 15, 18, 22, 26, 30, 19, 23, 27, 31 |
| resize_permC: dq 0, 1, 4, 5, 8, 9, 12, 13 |
| resize_permD: dq 2, 3, 6, 7, 10, 11, 14, 15 |
| resize_permE: dq 0, 2, 4, 6 |
| resize_shufA: db -1, 0, -1, 1, -1, 4, -1, 5, -1, 8, -1, 9, -1, 12, -1, 13 |
| resize_shufB: db -1, 2, -1, 3, -1, 6, -1, 7, -1, 10, -1, 11, -1, 14, -1, 15 |
| rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 |
| resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 |
| db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 |
| |
| prep_hv_shift: dq 6, 4 |
| put_bilin_h_rnd: dw 8, 8, 10, 10 |
| prep_mul: dw 16, 16, 4, 4 |
| put_8tap_h_rnd: dd 34, 40 |
| prep_8tap_rnd: dd 128 - (8192 << 8) |
| warp_8x8_rnd_h: dd 512, 2048 |
| warp_8x8_rnd_v: dd 262144, 65536 |
| warp_8x8t_rnd_v: dd 16384 - (8192 << 15) |
| avg_round: dw -16400, -16400, -16388, -16388 |
| w_avg_round: dd 128 + (8192 << 4), 32 + (8192 << 4) |
| mask_round: dd 512 + (8192 << 6), 128 + (8192 << 6) |
| w_mask_round: dd 128, 64 |
| bidir_shift: dw 6, 6, 4, 4 |
| |
| pb_64: times 4 db 64 |
| pw_m512: times 2 dw -512 |
| pw_2: times 2 dw 2 |
| pw_64: times 2 dw 64 |
| pd_32: dd 32 |
| pd_63: dd 63 |
| pd_128: dd 128 |
| pd_640: dd 640 |
| pd_2176: dd 2176 |
| pd_16384: dd 16384 |
| pd_0_4: dd 0, 4 |
| |
| %define pw_16 prep_mul |
| %define pd_512 warp_8x8_rnd_h |
| |
| %macro BASE_JMP_TABLE 3-* |
| %xdefine %1_%2_table (%%table - %3) |
| %xdefine %%base %1_%2 |
| %%table: |
| %rep %0 - 2 |
| dw %%base %+ _w%3 - %%base |
| %rotate 1 |
| %endrep |
| %endmacro |
| |
| %macro HV_JMP_TABLE 5-* |
| %xdefine %%prefix mangle(private_prefix %+ _%1_%2_16bpc_%3) |
| %xdefine %%base %1_%3 |
| %assign %%types %4 |
| %if %%types & 1 |
| %xdefine %1_%2_h_%3_table (%%h - %5) |
| %%h: |
| %rep %0 - 4 |
| dw %%prefix %+ .h_w%5 - %%base |
| %rotate 1 |
| %endrep |
| %rotate 4 |
| %endif |
| %if %%types & 2 |
| %xdefine %1_%2_v_%3_table (%%v - %5) |
| %%v: |
| %rep %0 - 4 |
| dw %%prefix %+ .v_w%5 - %%base |
| %rotate 1 |
| %endrep |
| %rotate 4 |
| %endif |
| %if %%types & 4 |
| %xdefine %1_%2_hv_%3_table (%%hv - %5) |
| %%hv: |
| %rep %0 - 4 |
| dw %%prefix %+ .hv_w%5 - %%base |
| %rotate 1 |
| %endrep |
| %endif |
| %endmacro |
| |
| %macro BIDIR_JMP_TABLE 2-* |
| %xdefine %1_%2_table (%%table - 2*%3) |
| %xdefine %%base %1_%2_table |
| %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) |
| %%table: |
| %rep %0 - 2 |
| dd %%prefix %+ .w%3 - %%base |
| %rotate 1 |
| %endrep |
| %endmacro |
| |
| %xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_16bpc_avx512icl.put) |
| %xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_16bpc_avx512icl.prep) |
| |
| BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128 |
| BIDIR_JMP_TABLE blend, avx512icl, 4, 8, 16, 32 |
| BIDIR_JMP_TABLE blend_v, avx512icl, 2, 4, 8, 16, 32 |
| BIDIR_JMP_TABLE blend_h, avx512icl, 2, 4, 8, 16, 32, 64, 128 |
| BASE_JMP_TABLE put, avx512icl, 2, 4, 8, 16, 32, 64, 128 |
| BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 |
| HV_JMP_TABLE put, bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128 |
| HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 |
| HV_JMP_TABLE put, 8tap, avx512icl, 2, 2, 4, 8, 16, 32, 64, 128 |
| HV_JMP_TABLE prep, 8tap, avx512icl, 2, 4, 8, 16, 32, 64, 128 |
| |
| %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX |
| |
| cextern mc_subpel_filters |
| %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) |
| |
| cextern mc_warp_filter |
| cextern obmc_masks_avx2 |
| cextern resize_filter |
| |
| SECTION .text |
| |
| %if WIN64 |
| DECLARE_REG_TMP 4 |
| %else |
| DECLARE_REG_TMP 8 |
| %endif |
| |
| INIT_ZMM avx512icl |
| cglobal put_bilin_16bpc, 4, 8, 13, dst, ds, src, ss, w, h, mxy |
| mov mxyd, r6m ; mx |
| lea r7, [put_avx512icl] |
| tzcnt t0d, wm |
| movifnidn hd, hm |
| test mxyd, mxyd |
| jnz .h |
| mov mxyd, r7m ; my |
| test mxyd, mxyd |
| jnz .v |
| .put: |
| movzx t0d, word [r7+t0*2+table_offset(put,)] |
| add t0, r7 |
| jmp t0 |
| .put_w2: |
| mov r6d, [srcq+ssq*0] |
| mov r7d, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mov [dstq+dsq*0], r6d |
| mov [dstq+dsq*1], r7d |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w2 |
| RET |
| .put_w4: |
| mov r6, [srcq+ssq*0] |
| mov r7, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mov [dstq+dsq*0], r6 |
| mov [dstq+dsq*1], r7 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w4 |
| RET |
| .put_w8: |
| movu xmm0, [srcq+ssq*0] |
| movu xmm1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mova [dstq+dsq*0], xmm0 |
| mova [dstq+dsq*1], xmm1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w8 |
| RET |
| .put_w16: |
| movu ym0, [srcq+ssq*0] |
| movu ym1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mova [dstq+dsq*0], ym0 |
| mova [dstq+dsq*1], ym1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w16 |
| RET |
| .put_w32: |
| movu m0, [srcq+ssq*0] |
| movu m1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mova [dstq+dsq*0], m0 |
| mova [dstq+dsq*1], m1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w32 |
| RET |
| .put_w64: |
| movu m0, [srcq+ssq*0+64*0] |
| movu m1, [srcq+ssq*0+64*1] |
| movu m2, [srcq+ssq*1+64*0] |
| movu m3, [srcq+ssq*1+64*1] |
| lea srcq, [srcq+ssq*2] |
| mova [dstq+dsq*0+64*0], m0 |
| mova [dstq+dsq*0+64*1], m1 |
| mova [dstq+dsq*1+64*0], m2 |
| mova [dstq+dsq*1+64*1], m3 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .put_w64 |
| RET |
| .put_w128: |
| movu m0, [srcq+64*0] |
| movu m1, [srcq+64*1] |
| movu m2, [srcq+64*2] |
| movu m3, [srcq+64*3] |
| add srcq, ssq |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| mova [dstq+64*2], m2 |
| mova [dstq+64*3], m3 |
| add dstq, dsq |
| dec hd |
| jg .put_w128 |
| RET |
| .h: |
| vpbroadcastw m5, mxyd |
| mov mxyd, r7m ; my |
| vpbroadcastd m4, [pw_16] |
| psubw m4, m5 |
| test mxyd, mxyd |
| jnz .hv |
| ; 12-bit is rounded twice so we can't use the same pmulhrsw approach as .v |
| movzx t0d, word [r7+t0*2+table_offset(put, _bilin_h)] |
| mov r6d, r8m ; bitdepth_max |
| add t0, r7 |
| shr r6d, 11 |
| vpbroadcastd m6, [r7-put_avx512icl+put_bilin_h_rnd+r6*4] |
| jmp t0 |
| .h_w2: |
| movq xmm1, [srcq+ssq*0] |
| movhps xmm1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| pmullw xmm0, xmm1, xm4 |
| psrlq xmm1, 16 |
| pmullw xmm1, xm5 |
| paddw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psrlw xmm0, 4 |
| movd [dstq+dsq*0], xmm0 |
| pextrd [dstq+dsq*1], xmm0, 2 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w2 |
| RET |
| .h_w4: |
| movq xmm0, [srcq+ssq*0+0] |
| movhps xmm0, [srcq+ssq*1+0] |
| movq xmm1, [srcq+ssq*0+2] |
| movhps xmm1, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| pmullw xmm0, xm4 |
| pmullw xmm1, xm5 |
| paddw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psrlw xmm0, 4 |
| movq [dstq+dsq*0], xmm0 |
| movhps [dstq+dsq*1], xmm0 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w4 |
| RET |
| .h_w8: |
| movu xm0, [srcq+ssq*0+0] |
| vinserti32x4 ym0, [srcq+ssq*1+0], 1 |
| movu xm1, [srcq+ssq*0+2] |
| vinserti32x4 ym1, [srcq+ssq*1+2], 1 |
| lea srcq, [srcq+ssq*2] |
| pmullw ym0, ym4 |
| pmullw ym1, ym5 |
| paddw ym0, ym6 |
| paddw ym0, ym1 |
| psrlw ym0, 4 |
| mova [dstq+dsq*0], xm0 |
| vextracti32x4 [dstq+dsq*1], ym0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w8 |
| RET |
| .h_w16: |
| movu ym0, [srcq+ssq*0+0] |
| vinserti32x8 m0, [srcq+ssq*1+0], 1 |
| movu ym1, [srcq+ssq*0+2] |
| vinserti32x8 m1, [srcq+ssq*1+2], 1 |
| lea srcq, [srcq+ssq*2] |
| pmullw m0, m4 |
| pmullw m1, m5 |
| paddw m0, m6 |
| paddw m0, m1 |
| psrlw m0, 4 |
| mova [dstq+dsq*0], ym0 |
| vextracti32x8 [dstq+dsq*1], m0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w16 |
| RET |
| .h_w32: |
| pmullw m0, m4, [srcq+ssq*0+0] |
| pmullw m2, m5, [srcq+ssq*0+2] |
| pmullw m1, m4, [srcq+ssq*1+0] |
| pmullw m3, m5, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| paddw m0, m6 |
| paddw m1, m6 |
| paddw m0, m2 |
| paddw m1, m3 |
| psrlw m0, 4 |
| psrlw m1, 4 |
| mova [dstq+dsq*0], m0 |
| mova [dstq+dsq*1], m1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w32 |
| RET |
| .h_w64: |
| pmullw m0, m4, [srcq+64*0+0] |
| pmullw m2, m5, [srcq+64*0+2] |
| pmullw m1, m4, [srcq+64*1+0] |
| pmullw m3, m5, [srcq+64*1+2] |
| add srcq, ssq |
| paddw m0, m6 |
| paddw m1, m6 |
| paddw m0, m2 |
| paddw m1, m3 |
| psrlw m0, 4 |
| psrlw m1, 4 |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| add dstq, dsq |
| dec hd |
| jg .h_w64 |
| RET |
| .h_w128: |
| pmullw m0, m4, [srcq+64*0+0] |
| pmullw m7, m5, [srcq+64*0+2] |
| pmullw m1, m4, [srcq+64*1+0] |
| pmullw m8, m5, [srcq+64*1+2] |
| pmullw m2, m4, [srcq+64*2+0] |
| pmullw m9, m5, [srcq+64*2+2] |
| pmullw m3, m4, [srcq+64*3+0] |
| pmullw m10, m5, [srcq+64*3+2] |
| add srcq, ssq |
| REPX {paddw x, m6}, m0, m1, m2, m3 |
| paddw m0, m7 |
| paddw m1, m8 |
| paddw m2, m9 |
| paddw m3, m10 |
| REPX {psrlw x, 4}, m0, m1, m2, m3 |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| mova [dstq+64*2], m2 |
| mova [dstq+64*3], m3 |
| add dstq, dsq |
| dec hd |
| jg .h_w128 |
| RET |
| .v: |
| movzx t0d, word [r7+t0*2+table_offset(put, _bilin_v)] |
| shl mxyd, 11 |
| vpbroadcastw m8, mxyd |
| add t0, r7 |
| jmp t0 |
| .v_w2: |
| movd xmm0, [srcq+ssq*0] |
| .v_w2_loop: |
| movd xmm1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| punpckldq xmm2, xmm0, xmm1 |
| movd xmm0, [srcq+ssq*0] |
| punpckldq xmm1, xmm0 |
| psubw xmm1, xmm2 |
| pmulhrsw xmm1, xm8 |
| paddw xmm1, xmm2 |
| movd [dstq+dsq*0], xmm1 |
| pextrd [dstq+dsq*1], xmm1, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w2_loop |
| RET |
| .v_w4: |
| movq xmm0, [srcq+ssq*0] |
| .v_w4_loop: |
| movq xmm1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| punpcklqdq xmm2, xmm0, xmm1 |
| movq xmm0, [srcq+ssq*0] |
| punpcklqdq xmm1, xmm0 |
| psubw xmm1, xmm2 |
| pmulhrsw xmm1, xm8 |
| paddw xmm1, xmm2 |
| movq [dstq+dsq*0], xmm1 |
| movhps [dstq+dsq*1], xmm1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w4_loop |
| RET |
| .v_w8: |
| movu xmm0, [srcq+ssq*0] |
| .v_w8_loop: |
| vbroadcasti128 ymm1, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| vpblendd ymm2, ymm0, ymm1, 0xf0 |
| vbroadcasti128 ymm0, [srcq+ssq*0] |
| vpblendd ymm1, ymm0, 0xf0 |
| psubw ymm1, ymm2 |
| pmulhrsw ymm1, ym8 |
| paddw ymm1, ymm2 |
| mova [dstq+dsq*0], xmm1 |
| vextracti128 [dstq+dsq*1], ymm1, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w8_loop |
| vzeroupper |
| RET |
| .v_w16: |
| movu ym0, [srcq+ssq*0] |
| .v_w16_loop: |
| movu ym3, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| psubw ym1, ym3, ym0 |
| pmulhrsw ym1, ym8 |
| paddw ym1, ym0 |
| movu ym0, [srcq+ssq*0] |
| psubw ym2, ym0, ym3 |
| pmulhrsw ym2, ym8 |
| paddw ym2, ym3 |
| mova [dstq+dsq*0], ym1 |
| mova [dstq+dsq*1], ym2 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w16_loop |
| RET |
| .v_w32: |
| movu m0, [srcq+ssq*0] |
| .v_w32_loop: |
| movu m3, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| psubw m1, m3, m0 |
| pmulhrsw m1, m8 |
| paddw m1, m0 |
| movu m0, [srcq+ssq*0] |
| psubw m2, m0, m3 |
| pmulhrsw m2, m8 |
| paddw m2, m3 |
| mova [dstq+dsq*0], m1 |
| mova [dstq+dsq*1], m2 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w32_loop |
| RET |
| .v_w64: |
| movu m0, [srcq+ssq*0+64*0] |
| movu m1, [srcq+ssq*0+64*1] |
| .v_w64_loop: |
| movu m2, [srcq+ssq*1+64*0] |
| movu m3, [srcq+ssq*1+64*1] |
| lea srcq, [srcq+ssq*2] |
| psubw m4, m2, m0 |
| pmulhrsw m4, m8 |
| paddw m4, m0 |
| movu m0, [srcq+ssq*0+64*0] |
| psubw m5, m3, m1 |
| pmulhrsw m5, m8 |
| paddw m5, m1 |
| movu m1, [srcq+ssq*0+64*1] |
| psubw m6, m0, m2 |
| pmulhrsw m6, m8 |
| psubw m7, m1, m3 |
| pmulhrsw m7, m8 |
| mova [dstq+dsq*0+64*0], m4 |
| mova [dstq+dsq*0+64*1], m5 |
| paddw m6, m2 |
| paddw m7, m3 |
| mova [dstq+dsq*1+64*0], m6 |
| mova [dstq+dsq*1+64*1], m7 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w64_loop |
| RET |
| .v_w128: |
| movu m0, [srcq+ssq*0+64*0] |
| movu m1, [srcq+ssq*0+64*1] |
| movu m2, [srcq+ssq*0+64*2] |
| movu m3, [srcq+ssq*0+64*3] |
| .v_w128_loop: |
| movu m4, [srcq+ssq*1+64*0] |
| movu m5, [srcq+ssq*1+64*1] |
| movu m6, [srcq+ssq*1+64*2] |
| movu m7, [srcq+ssq*1+64*3] |
| lea srcq, [srcq+ssq*2] |
| psubw m9, m4, m0 |
| pmulhrsw m9, m8 |
| paddw m9, m0 |
| movu m0, [srcq+ssq*0+64*0] |
| psubw m10, m5, m1 |
| pmulhrsw m10, m8 |
| paddw m10, m1 |
| movu m1, [srcq+ssq*0+64*1] |
| psubw m11, m6, m2 |
| pmulhrsw m11, m8 |
| paddw m11, m2 |
| movu m2, [srcq+ssq*0+64*2] |
| psubw m12, m7, m3 |
| pmulhrsw m12, m8 |
| paddw m12, m3 |
| movu m3, [srcq+ssq*0+64*3] |
| mova [dstq+dsq*0+64*0], m9 |
| psubw m9, m0, m4 |
| pmulhrsw m9, m8 |
| mova [dstq+dsq*0+64*1], m10 |
| psubw m10, m1, m5 |
| pmulhrsw m10, m8 |
| mova [dstq+dsq*0+64*2], m11 |
| psubw m11, m2, m6 |
| pmulhrsw m11, m8 |
| mova [dstq+dsq*0+64*3], m12 |
| psubw m12, m3, m7 |
| pmulhrsw m12, m8 |
| paddw m9, m4 |
| paddw m10, m5 |
| mova [dstq+dsq*1+64*0], m9 |
| mova [dstq+dsq*1+64*1], m10 |
| paddw m11, m6 |
| paddw m12, m7 |
| mova [dstq+dsq*1+64*2], m11 |
| mova [dstq+dsq*1+64*3], m12 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w128_loop |
| RET |
| .hv: |
| movzx t0d, word [r7+t0*2+table_offset(put, _bilin_hv)] |
| shl mxyd, 11 |
| vpbroadcastd m6, [pw_2] |
| vpbroadcastw m7, mxyd |
| vpbroadcastd m8, [pw_8192] |
| add t0, r7 |
| test dword r8m, 0x800 |
| jnz .hv_12bpc |
| psllw m4, 2 |
| psllw m5, 2 |
| vpbroadcastd m8, [pw_2048] |
| .hv_12bpc: |
| jmp t0 |
| .hv_w2: |
| vpbroadcastq xmm1, [srcq+ssq*0] |
| pmullw xmm0, xmm1, xm4 |
| psrlq xmm1, 16 |
| pmullw xmm1, xm5 |
| paddw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psrlw xmm0, 2 |
| .hv_w2_loop: |
| movq xmm2, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| movhps xmm2, [srcq+ssq*0] |
| pmullw xmm1, xmm2, xm4 |
| psrlq xmm2, 16 |
| pmullw xmm2, xm5 |
| paddw xmm1, xm6 |
| paddw xmm1, xmm2 |
| psrlw xmm1, 2 ; 1 _ 2 _ |
| shufpd xmm2, xmm0, xmm1, 0x01 ; 0 _ 1 _ |
| mova xmm0, xmm1 |
| psubw xmm1, xmm2 |
| paddw xmm1, xmm1 |
| pmulhw xmm1, xm7 |
| paddw xmm1, xmm2 |
| pmulhrsw xmm1, xm8 |
| movd [dstq+dsq*0], xmm1 |
| pextrd [dstq+dsq*1], xmm1, 2 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w2_loop |
| RET |
| .hv_w4: |
| pmullw xmm0, xm4, [srcq+ssq*0-8] |
| pmullw xmm1, xm5, [srcq+ssq*0-6] |
| paddw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psrlw xmm0, 2 |
| .hv_w4_loop: |
| movq xmm1, [srcq+ssq*1+0] |
| movq xmm2, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| movhps xmm1, [srcq+ssq*0+0] |
| movhps xmm2, [srcq+ssq*0+2] |
| pmullw xmm1, xm4 |
| pmullw xmm2, xm5 |
| paddw xmm1, xm6 |
| paddw xmm1, xmm2 |
| psrlw xmm1, 2 ; 1 2 |
| shufpd xmm2, xmm0, xmm1, 0x01 ; 0 1 |
| mova xmm0, xmm1 |
| psubw xmm1, xmm2 |
| paddw xmm1, xmm1 |
| pmulhw xmm1, xm7 |
| paddw xmm1, xmm2 |
| pmulhrsw xmm1, xm8 |
| movq [dstq+dsq*0], xmm1 |
| movhps [dstq+dsq*1], xmm1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w4_loop |
| RET |
| .hv_w8: |
| pmullw xmm0, xm4, [srcq+ssq*0+0] |
| pmullw xmm1, xm5, [srcq+ssq*0+2] |
| paddw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psrlw xmm0, 2 |
| vinserti32x4 ym0, xmm0, 1 |
| .hv_w8_loop: |
| movu xm1, [srcq+ssq*1+0] |
| movu xm2, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x4 ym1, [srcq+ssq*0+0], 1 |
| vinserti32x4 ym2, [srcq+ssq*0+2], 1 |
| pmullw ym1, ym4 |
| pmullw ym2, ym5 |
| paddw ym1, ym6 |
| paddw ym1, ym2 |
| psrlw ym1, 2 ; 1 2 |
| vshufi32x4 ym2, ym0, ym1, 0x01 ; 0 1 |
| mova ym0, ym1 |
| psubw ym1, ym2 |
| paddw ym1, ym1 |
| pmulhw ym1, ym7 |
| paddw ym1, ym2 |
| pmulhrsw ym1, ym8 |
| mova [dstq+dsq*0], xm1 |
| vextracti32x4 [dstq+dsq*1], ym1, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w8_loop |
| RET |
| .hv_w16: |
| pmullw ym0, ym4, [srcq+ssq*0+0] |
| pmullw ym1, ym5, [srcq+ssq*0+2] |
| paddw ym0, ym6 |
| paddw ym0, ym1 |
| psrlw ym0, 2 |
| vinserti32x8 m0, ym0, 1 |
| .hv_w16_loop: |
| movu ym1, [srcq+ssq*1+0] |
| movu ym2, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x8 m1, [srcq+ssq*0+0], 1 |
| vinserti32x8 m2, [srcq+ssq*0+2], 1 |
| pmullw m1, m4 |
| pmullw m2, m5 |
| paddw m1, m6 |
| paddw m1, m2 |
| psrlw m1, 2 ; 1 2 |
| vshufi32x4 m2, m0, m1, q1032 ; 0 1 |
| mova m0, m1 |
| psubw m1, m2 |
| paddw m1, m1 |
| pmulhw m1, m7 |
| paddw m1, m2 |
| pmulhrsw m1, m8 |
| mova [dstq+dsq*0], ym1 |
| vextracti32x8 [dstq+dsq*1], m1, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w16_loop |
| RET |
| .hv_w32: |
| .hv_w64: |
| .hv_w128: |
| movifnidn wd, wm |
| lea r6d, [hq+wq*8-256] |
| mov r4, srcq |
| mov r7, dstq |
| .hv_w32_loop0: |
| pmullw m0, m4, [srcq+ssq*0+0] |
| pmullw m1, m5, [srcq+ssq*0+2] |
| paddw m0, m6 |
| paddw m0, m1 |
| psrlw m0, 2 |
| .hv_w32_loop: |
| pmullw m3, m4, [srcq+ssq*1+0] |
| pmullw m1, m5, [srcq+ssq*1+2] |
| lea srcq, [srcq+ssq*2] |
| paddw m3, m6 |
| paddw m3, m1 |
| psrlw m3, 2 |
| psubw m1, m3, m0 |
| paddw m1, m1 |
| pmulhw m1, m7 |
| paddw m1, m0 |
| pmullw m0, m4, [srcq+ssq*0+0] |
| pmullw m2, m5, [srcq+ssq*0+2] |
| paddw m0, m6 |
| paddw m0, m2 |
| psrlw m0, 2 |
| psubw m2, m0, m3 |
| paddw m2, m2 |
| pmulhw m2, m7 |
| paddw m2, m3 |
| pmulhrsw m1, m8 |
| pmulhrsw m2, m8 |
| mova [dstq+dsq*0], m1 |
| mova [dstq+dsq*1], m2 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w32_loop |
| add r4, 64 |
| add r7, 64 |
| movzx hd, r6b |
| mov srcq, r4 |
| mov dstq, r7 |
| sub r6d, 1<<8 |
| jg .hv_w32_loop0 |
| RET |
| |
| cglobal prep_bilin_16bpc, 3, 7, 16, tmp, src, stride, w, h, mxy, stride3 |
| movifnidn mxyd, r5m ; mx |
| lea r6, [prep_avx512icl] |
| tzcnt wd, wm |
| movifnidn hd, hm |
| test mxyd, mxyd |
| jnz .h |
| mov mxyd, r6m ; my |
| test mxyd, mxyd |
| jnz .v |
| .prep: |
| movzx wd, word [r6+wq*2+table_offset(prep,)] |
| mov r5d, r7m ; bitdepth_max |
| vpbroadcastd m5, [r6-prep_avx512icl+pw_8192] |
| add wq, r6 |
| shr r5d, 11 |
| vpbroadcastd m4, [r6-prep_avx512icl+prep_mul+r5*4] |
| lea stride3q, [strideq*3] |
| jmp wq |
| .prep_w4: |
| movq xmm0, [srcq+strideq*0] |
| movhps xmm0, [srcq+strideq*1] |
| vpbroadcastq ymm1, [srcq+strideq*2] |
| vpbroadcastq ymm2, [srcq+stride3q ] |
| lea srcq, [srcq+strideq*4] |
| vpblendd ymm0, ymm1, 0x30 |
| vpblendd ymm0, ymm2, 0xc0 |
| pmullw ymm0, ym4 |
| psubw ymm0, ym5 |
| mova [tmpq], ymm0 |
| add tmpq, 32 |
| sub hd, 4 |
| jg .prep_w4 |
| vzeroupper |
| RET |
| .prep_w8: |
| movu xm0, [srcq+strideq*0] |
| vinserti32x4 ym0, [srcq+strideq*1], 1 |
| vinserti32x4 m0, [srcq+strideq*2], 2 |
| vinserti32x4 m0, [srcq+stride3q ], 3 |
| lea srcq, [srcq+strideq*4] |
| pmullw m0, m4 |
| psubw m0, m5 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| sub hd, 4 |
| jg .prep_w8 |
| RET |
| .prep_w16: |
| movu ym0, [srcq+strideq*0] |
| vinserti32x8 m0, [srcq+strideq*1], 1 |
| movu ym1, [srcq+strideq*2] |
| vinserti32x8 m1, [srcq+stride3q ], 1 |
| lea srcq, [srcq+strideq*4] |
| pmullw m0, m4 |
| pmullw m1, m4 |
| psubw m0, m5 |
| psubw m1, m5 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| add tmpq, 64*2 |
| sub hd, 4 |
| jg .prep_w16 |
| RET |
| .prep_w32: |
| pmullw m0, m4, [srcq+strideq*0] |
| pmullw m1, m4, [srcq+strideq*1] |
| pmullw m2, m4, [srcq+strideq*2] |
| pmullw m3, m4, [srcq+stride3q ] |
| lea srcq, [srcq+strideq*4] |
| REPX {psubw x, m5}, m0, m1, m2, m3 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| mova [tmpq+64*2], m2 |
| mova [tmpq+64*3], m3 |
| add tmpq, 64*4 |
| sub hd, 4 |
| jg .prep_w32 |
| RET |
| .prep_w64: |
| pmullw m0, m4, [srcq+strideq*0+64*0] |
| pmullw m1, m4, [srcq+strideq*0+64*1] |
| pmullw m2, m4, [srcq+strideq*1+64*0] |
| pmullw m3, m4, [srcq+strideq*1+64*1] |
| lea srcq, [srcq+strideq*2] |
| REPX {psubw x, m5}, m0, m1, m2, m3 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| mova [tmpq+64*2], m2 |
| mova [tmpq+64*3], m3 |
| add tmpq, 64*4 |
| sub hd, 2 |
| jg .prep_w64 |
| RET |
| .prep_w128: |
| pmullw m0, m4, [srcq+64*0] |
| pmullw m1, m4, [srcq+64*1] |
| pmullw m2, m4, [srcq+64*2] |
| pmullw m3, m4, [srcq+64*3] |
| add srcq, strideq |
| REPX {psubw x, m5}, m0, m1, m2, m3 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| mova [tmpq+64*2], m2 |
| mova [tmpq+64*3], m3 |
| add tmpq, 64*4 |
| dec hd |
| jg .prep_w128 |
| RET |
| .h: |
| vpbroadcastw m5, mxyd |
| mov mxyd, r6m ; my |
| vpbroadcastd m4, [pw_16] |
| vpbroadcastd m6, [pw_32766] |
| psubw m4, m5 |
| test dword r7m, 0x800 |
| jnz .h_12bpc |
| psllw m4, 2 |
| psllw m5, 2 |
| .h_12bpc: |
| test mxyd, mxyd |
| jnz .hv |
| movzx wd, word [r6+wq*2+table_offset(prep, _bilin_h)] |
| add wq, r6 |
| lea stride3q, [strideq*3] |
| jmp wq |
| .h_w4: |
| movu xm1, [srcq+strideq*0] |
| vinserti32x4 ym1, [srcq+strideq*2], 1 |
| movu xm2, [srcq+strideq*1] |
| vinserti32x4 ym2, [srcq+stride3q ], 1 |
| lea srcq, [srcq+strideq*4] |
| punpcklqdq ym0, ym1, ym2 |
| psrldq ym1, 2 |
| psrldq ym2, 2 |
| pmullw ym0, ym4 |
| punpcklqdq ym1, ym2 |
| pmullw ym1, ym5 |
| psubw ym0, ym6 |
| paddw ym0, ym1 |
| psraw ym0, 2 |
| mova [tmpq], ym0 |
| add tmpq, 32 |
| sub hd, 4 |
| jg .h_w4 |
| RET |
| .h_w8: |
| movu xm0, [srcq+strideq*0+0] |
| movu xm1, [srcq+strideq*0+2] |
| vinserti32x4 ym0, [srcq+strideq*1+0], 1 |
| vinserti32x4 ym1, [srcq+strideq*1+2], 1 |
| vinserti32x4 m0, [srcq+strideq*2+0], 2 |
| vinserti32x4 m1, [srcq+strideq*2+2], 2 |
| vinserti32x4 m0, [srcq+stride3q +0], 3 |
| vinserti32x4 m1, [srcq+stride3q +2], 3 |
| lea srcq, [srcq+strideq*4] |
| pmullw m0, m4 |
| pmullw m1, m5 |
| psubw m0, m6 |
| paddw m0, m1 |
| psraw m0, 2 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| sub hd, 4 |
| jg .h_w8 |
| RET |
| .h_w16: |
| movu ym0, [srcq+strideq*0+0] |
| vinserti32x8 m0, [srcq+strideq*1+0], 1 |
| movu ym1, [srcq+strideq*0+2] |
| vinserti32x8 m1, [srcq+strideq*1+2], 1 |
| lea srcq, [srcq+strideq*2] |
| pmullw m0, m4 |
| pmullw m1, m5 |
| psubw m0, m6 |
| paddw m0, m1 |
| psraw m0, 2 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| sub hd, 2 |
| jg .h_w16 |
| RET |
| .h_w32: |
| pmullw m0, m4, [srcq+strideq*0+0] |
| pmullw m2, m5, [srcq+strideq*0+2] |
| pmullw m1, m4, [srcq+strideq*1+0] |
| pmullw m3, m5, [srcq+strideq*1+2] |
| lea srcq, [srcq+strideq*2] |
| psubw m0, m6 |
| psubw m1, m6 |
| paddw m0, m2 |
| paddw m1, m3 |
| psraw m0, 2 |
| psraw m1, 2 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| add tmpq, 64*2 |
| sub hd, 2 |
| jg .h_w32 |
| RET |
| .h_w64: |
| pmullw m0, m4, [srcq+ 0] |
| pmullw m2, m5, [srcq+ 2] |
| pmullw m1, m4, [srcq+64] |
| pmullw m3, m5, [srcq+66] |
| add srcq, strideq |
| psubw m0, m6 |
| psubw m1, m6 |
| paddw m0, m2 |
| paddw m1, m3 |
| psraw m0, 2 |
| psraw m1, 2 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| add tmpq, 64*2 |
| dec hd |
| jg .h_w64 |
| RET |
| .h_w128: |
| pmullw m0, m4, [srcq+ 0] |
| pmullw m7, m5, [srcq+ 2] |
| pmullw m1, m4, [srcq+ 64] |
| pmullw m8, m5, [srcq+ 66] |
| pmullw m2, m4, [srcq+128] |
| pmullw m9, m5, [srcq+130] |
| pmullw m3, m4, [srcq+192] |
| pmullw m10, m5, [srcq+194] |
| add srcq, strideq |
| REPX {psubw x, m6}, m0, m1, m2, m3 |
| paddw m0, m7 |
| paddw m1, m8 |
| paddw m2, m9 |
| paddw m3, m10 |
| REPX {psraw x, 2}, m0, m1, m2, m3 |
| mova [tmpq+64*0], m0 |
| mova [tmpq+64*1], m1 |
| mova [tmpq+64*2], m2 |
| mova [tmpq+64*3], m3 |
| add tmpq, 64*4 |
| dec hd |
| jg .h_w128 |
| RET |
| .v: |
| movzx wd, word [r6+wq*2+table_offset(prep, _bilin_v)] |
| vpbroadcastw m9, mxyd |
| vpbroadcastd m8, [pw_16] |
| vpbroadcastd m10, [pw_32766] |
| add wq, r6 |
| lea stride3q, [strideq*3] |
| psubw m8, m9 |
| test dword r7m, 0x800 |
| jnz .v_12bpc |
| psllw m8, 2 |
| psllw m9, 2 |
| .v_12bpc: |
| jmp wq |
| .v_w4: |
| movq xmm0, [srcq+strideq*0] |
| .v_w4_loop: |
| vpbroadcastq xmm2, [srcq+strideq*1] |
| vpbroadcastq ymm1, [srcq+strideq*2] |
| vpbroadcastq ymm3, [srcq+stride3q ] |
| lea srcq, [srcq+strideq*4] |
| vpblendd ymm2, ymm1, 0x30 |
| vpblendd ymm2, ymm3, 0xc0 |
| vpblendd ymm1, ymm2, ymm0, 0x03 ; 0 1 2 3 |
| movq xmm0, [srcq+strideq*0] |
| valignq ymm2, ymm0, ymm2, 1 ; 1 2 3 4 |
| pmullw ymm1, ym8 |
| pmullw ymm2, ym9 |
| psubw ymm1, ym10 |
| paddw ymm1, ymm2 |
| psraw ymm1, 2 |
| mova [tmpq], ymm1 |
| add tmpq, 32 |
| sub hd, 4 |
| jg .v_w4_loop |
| vzeroupper |
| RET |
| .v_w8: |
| movu xm0, [srcq+strideq*0] |
| .v_w8_loop: |
| vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 |
| vinserti32x4 m1, [srcq+strideq*2], 2 |
| vinserti32x4 m1, [srcq+stride3q ], 3 ; 0 1 2 3 |
| lea srcq, [srcq+strideq*4] |
| movu xm0, [srcq+strideq*0] |
| valignq m2, m0, m1, 2 ; 1 2 3 4 |
| pmullw m1, m8 |
| pmullw m2, m9 |
| psubw m1, m10 |
| paddw m1, m2 |
| psraw m1, 2 |
| mova [tmpq], m1 |
| add tmpq, 64 |
| sub hd, 4 |
| jg .v_w8_loop |
| RET |
| .v_w16: |
| movu ym0, [srcq+strideq*0] |
| .v_w16_loop: |
| vinserti32x8 m1, m0, [srcq+strideq*1], 1 ; 0 1 |
| movu ym3, [srcq+strideq*2] |
| vinserti32x8 m2, m3, [srcq+stride3q ], 1 ; 2 3 |
| lea srcq, [srcq+strideq*4] |
| movu ym0, [srcq+strideq*0] |
| vshufi32x4 m3, m1, m3, q1032 ; 1 2 |
| vshufi32x4 m4, m2, m0, q1032 ; 3 4 |
| pmullw m1, m8 |
| pmullw m2, m8 |
| pmullw m3, m9 |
| pmullw m4, m9 |
| psubw m1, m10 |
| psubw m2, m10 |
| paddw m1, m3 |
| paddw m2, m4 |
| psraw m1, 2 |
| psraw m2, 2 |
| mova [tmpq+64*0], m1 |
| mova [tmpq+64*1], m2 |
| add tmpq, 64*2 |
| sub hd, 4 |
| jg .v_w16_loop |
| RET |
| .v_w32: |
| movu m0, [srcq+strideq*0] |
| .v_w32_loop: |
| movu m3, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| pmullw m1, m8, m0 |
| movu m0, [srcq+strideq*0] |
| pmullw m2, m8, m3 |
| pmullw m3, m9 |
| pmullw m4, m9, m0 |
| psubw m1, m10 |
| psubw m2, m10 |
| paddw m1, m3 |
| paddw m2, m4 |
| psraw m1, 2 |
| psraw m2, 2 |
| mova [tmpq+64*0], m1 |
| mova [tmpq+64*1], m2 |
| add tmpq, 64*2 |
| sub hd, 2 |
| jg .v_w32_loop |
| RET |
| .v_w64: |
| movu m0, [srcq+64*0] |
| movu m1, [srcq+64*1] |
| .v_w64_loop: |
| add srcq, strideq |
| pmullw m2, m8, m0 |
| movu m0, [srcq+64*0] |
| pmullw m3, m8, m1 |
| movu m1, [srcq+64*1] |
| pmullw m4, m9, m0 |
| pmullw m5, m9, m1 |
| psubw m2, m10 |
| psubw m3, m10 |
| paddw m2, m4 |
| paddw m3, m5 |
| psraw m2, 2 |
| psraw m3, 2 |
| mova [tmpq+64*0], m2 |
| mova [tmpq+64*1], m3 |
| add tmpq, 64*2 |
| dec hd |
| jg .v_w64_loop |
| RET |
| .v_w128: |
| movu m0, [srcq+64*0] |
| movu m1, [srcq+64*1] |
| movu m2, [srcq+64*2] |
| movu m3, [srcq+64*3] |
| .v_w128_loop: |
| add srcq, strideq |
| pmullw m4, m8, m0 |
| movu m0, [srcq+64*0] |
| pmullw m5, m8, m1 |
| movu m1, [srcq+64*1] |
| pmullw m6, m8, m2 |
| movu m2, [srcq+64*2] |
| pmullw m7, m8, m3 |
| movu m3, [srcq+64*3] |
| pmullw m11, m9, m0 |
| pmullw m12, m9, m1 |
| pmullw m13, m9, m2 |
| pmullw m14, m9, m3 |
| REPX {psubw x, m10}, m4, m5, m6, m7 |
| paddw m4, m11 |
| paddw m5, m12 |
| paddw m6, m13 |
| paddw m7, m14 |
| REPX {psraw x, 2}, m4, m5, m6, m7 |
| mova [tmpq+64*0], m4 |
| mova [tmpq+64*1], m5 |
| mova [tmpq+64*2], m6 |
| mova [tmpq+64*3], m7 |
| add tmpq, 64*4 |
| dec hd |
| jg .v_w128_loop |
| RET |
| .hv: |
| movzx wd, word [r6+wq*2+table_offset(prep, _bilin_hv)] |
| shl mxyd, 11 |
| vpbroadcastw m7, mxyd |
| add wq, r6 |
| lea stride3q, [strideq*3] |
| jmp wq |
| .hv_w4: |
| movq xmm0, [srcq+strideq*0+0] |
| movq xmm1, [srcq+strideq*0+2] |
| pmullw xmm0, xm4 |
| pmullw xmm1, xm5 |
| psubw xmm0, xm6 |
| paddw xmm0, xmm1 |
| psraw xmm0, 2 |
| vpbroadcastq ym0, xmm0 |
| .hv_w4_loop: |
| movu xm1, [srcq+strideq*1] |
| vinserti128 ym1, [srcq+stride3q ], 1 |
| movu xm2, [srcq+strideq*2] |
| lea srcq, [srcq+strideq*4] |
| vinserti128 ym2, [srcq+strideq*0], 1 |
| punpcklqdq ym3, ym1, ym2 |
| psrldq ym1, 2 |
| psrldq ym2, 2 |
| pmullw ym3, ym4 |
| punpcklqdq ym1, ym2 |
| pmullw ym1, ym5 |
| psubw ym3, ym6 |
| paddw ym1, ym3 |
| psraw ym1, 2 ; 1 2 3 4 |
| valignq ym2, ym1, ym0, 3 ; 0 1 2 3 |
| mova ym0, ym1 |
| psubw ym1, ym2 |
| pmulhrsw ym1, ym7 |
| paddw ym1, ym2 |
| mova [tmpq], ym1 |
| add tmpq, 32 |
| sub hd, 4 |
| jg .hv_w4_loop |
| RET |
| .hv_w8: |
| pmullw xm0, xm4, [srcq+strideq*0+0] |
| pmullw xm1, xm5, [srcq+strideq*0+2] |
| psubw xm0, xm6 |
| paddw xm0, xm1 |
| psraw xm0, 2 |
| vinserti32x4 m0, xm0, 3 |
| .hv_w8_loop: |
| movu xm1, [srcq+strideq*1+0] |
| movu xm2, [srcq+strideq*1+2] |
| vinserti32x4 ym1, [srcq+strideq*2+0], 1 |
| vinserti32x4 ym2, [srcq+strideq*2+2], 1 |
| vinserti32x4 m1, [srcq+stride3q +0], 2 |
| vinserti32x4 m2, [srcq+stride3q +2], 2 |
| lea srcq, [srcq+strideq*4] |
| vinserti32x4 m1, [srcq+strideq*0+0], 3 |
| vinserti32x4 m2, [srcq+strideq*0+2], 3 |
| pmullw m1, m4 |
| pmullw m2, m5 |
| psubw m1, m6 |
| paddw m1, m2 |
| psraw m1, 2 ; 1 2 3 4 |
| valignq m2, m1, m0, 6 ; 0 1 2 3 |
| mova m0, m1 |
| psubw m1, m2 |
| pmulhrsw m1, m7 |
| paddw m1, m2 |
| mova [tmpq], m1 |
| add tmpq, 64 |
| sub hd, 4 |
| jg .hv_w8_loop |
| RET |
| .hv_w16: |
| pmullw ym0, ym4, [srcq+strideq*0+0] |
| pmullw ym1, ym5, [srcq+strideq*0+2] |
| psubw ym0, ym6 |
| paddw ym0, ym1 |
| psraw ym0, 2 |
| vinserti32x8 m0, ym0, 1 |
| .hv_w16_loop: |
| movu ym1, [srcq+strideq*1+0] |
| movu ym2, [srcq+strideq*1+2] |
| lea srcq, [srcq+strideq*2] |
| vinserti32x8 m1, [srcq+strideq*0+0], 1 |
| vinserti32x8 m2, [srcq+strideq*0+2], 1 |
| pmullw m1, m4 |
| pmullw m2, m5 |
| psubw m1, m6 |
| paddw m1, m2 |
| psraw m1, 2 ; 1 2 |
| vshufi32x4 m2, m0, m1, q1032 ; 0 1 |
| mova m0, m1 |
| psubw m1, m2 |
| pmulhrsw m1, m7 |
| paddw m1, m2 |
| mova [tmpq], m1 |
| add tmpq, 64 |
| sub hd, 2 |
| jg .hv_w16_loop |
| RET |
| .hv_w32: |
| pmullw m0, m4, [srcq+strideq*0+0] |
| pmullw m1, m5, [srcq+strideq*0+2] |
| psubw m0, m6 |
| paddw m0, m1 |
| psraw m0, 2 |
| .hv_w32_loop: |
| pmullw m3, m4, [srcq+strideq*1+0] |
| pmullw m1, m5, [srcq+strideq*1+2] |
| lea srcq, [srcq+strideq*2] |
| psubw m3, m6 |
| paddw m3, m1 |
| psraw m3, 2 |
| psubw m1, m3, m0 |
| pmulhrsw m1, m7 |
| paddw m1, m0 |
| pmullw m0, m4, [srcq+strideq*0+0] |
| pmullw m2, m5, [srcq+strideq*0+2] |
| psubw m0, m6 |
| paddw m0, m2 |
| psraw m0, 2 |
| psubw m2, m0, m3 |
| pmulhrsw m2, m7 |
| paddw m2, m3 |
| mova [tmpq+64*0], m1 |
| mova [tmpq+64*1], m2 |
| add tmpq, 64*2 |
| sub hd, 2 |
| jg .hv_w32_loop |
| RET |
| .hv_w64: |
| pmullw m0, m4, [srcq+ 0] |
| pmullw m2, m5, [srcq+ 2] |
| pmullw m1, m4, [srcq+64] |
| pmullw m3, m5, [srcq+66] |
| psubw m0, m6 |
| psubw m1, m6 |
| paddw m0, m2 |
| paddw m1, m3 |
| psraw m0, 2 |
| psraw m1, 2 |
| .hv_w64_loop: |
| add srcq, strideq |
| pmullw m2, m4, [srcq+ 0] |
| pmullw m8, m5, [srcq+ 2] |
| pmullw m3, m4, [srcq+64] |
| pmullw m9, m5, [srcq+66] |
| psubw m2, m6 |
| psubw m3, m6 |
| paddw m2, m8 |
| paddw m3, m9 |
| psraw m2, 2 |
| psraw m3, 2 |
| psubw m8, m2, m0 |
| psubw m9, m3, m1 |
| pmulhrsw m8, m7 |
| pmulhrsw m9, m7 |
| paddw m8, m0 |
| mova m0, m2 |
| paddw m9, m1 |
| mova m1, m3 |
| mova [tmpq+64*0], m8 |
| mova [tmpq+64*1], m9 |
| add tmpq, 64*2 |
| dec hd |
| jg .hv_w64_loop |
| RET |
| .hv_w128: |
| pmullw m0, m4, [srcq+ 0] |
| pmullw m8, m5, [srcq+ 2] |
| pmullw m1, m4, [srcq+ 64] |
| pmullw m9, m5, [srcq+ 66] |
| pmullw m2, m4, [srcq+128] |
| pmullw m10, m5, [srcq+130] |
| pmullw m3, m4, [srcq+192] |
| pmullw m11, m5, [srcq+194] |
| REPX {psubw x, m6}, m0, m1, m2, m3 |
| paddw m0, m8 |
| paddw m1, m9 |
| paddw m2, m10 |
| paddw m3, m11 |
| REPX {psraw x, 2}, m0, m1, m2, m3 |
| .hv_w128_loop: |
| add srcq, strideq |
| pmullw m8, m4, [srcq+ 0] |
| pmullw m12, m5, [srcq+ 2] |
| pmullw m9, m4, [srcq+ 64] |
| pmullw m13, m5, [srcq+ 66] |
| pmullw m10, m4, [srcq+128] |
| pmullw m14, m5, [srcq+130] |
| pmullw m11, m4, [srcq+192] |
| pmullw m15, m5, [srcq+194] |
| REPX {psubw x, m6}, m8, m9, m10, m11 |
| paddw m8, m12 |
| paddw m9, m13 |
| paddw m10, m14 |
| paddw m11, m15 |
| REPX {psraw x, 2}, m8, m9, m10, m11 |
| psubw m12, m8, m0 |
| psubw m13, m9, m1 |
| psubw m14, m10, m2 |
| psubw m15, m11, m3 |
| REPX {pmulhrsw x, m7}, m12, m13, m14, m15 |
| paddw m12, m0 |
| mova m0, m8 |
| paddw m13, m1 |
| mova m1, m9 |
| mova [tmpq+64*0], m12 |
| mova [tmpq+64*1], m13 |
| paddw m14, m2 |
| mova m2, m10 |
| paddw m15, m3 |
| mova m3, m11 |
| mova [tmpq+64*2], m14 |
| mova [tmpq+64*3], m15 |
| add tmpq, 64*4 |
| dec hd |
| jg .hv_w128_loop |
| RET |
| |
| ; int8_t subpel_filters[5][15][8] |
| %assign FILTER_REGULAR (0*15 << 16) | 3*15 |
| %assign FILTER_SMOOTH (1*15 << 16) | 4*15 |
| %assign FILTER_SHARP (2*15 << 16) | 3*15 |
| |
| %macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v |
| cglobal %1_8tap_%2_16bpc |
| mov t0d, FILTER_%3 |
| %ifidn %3, %4 |
| mov t1d, t0d |
| %else |
| mov t1d, FILTER_%4 |
| %endif |
| %ifnidn %2, regular ; skip the jump in the last filter |
| jmp mangle(private_prefix %+ _%1_8tap_16bpc %+ SUFFIX) |
| %endif |
| %endmacro |
| |
| %if WIN64 |
| DECLARE_REG_TMP 4, 5 |
| %define buf rsp+stack_offset+8 ; shadow space |
| %else |
| DECLARE_REG_TMP 7, 8 |
| %define buf rsp-40 ; red zone |
| %endif |
| |
| MC_8TAP_FN put, sharp, SHARP, SHARP |
| MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH |
| MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP |
| MC_8TAP_FN put, smooth, SMOOTH, SMOOTH |
| MC_8TAP_FN put, sharp_regular, SHARP, REGULAR |
| MC_8TAP_FN put, regular_sharp, REGULAR, SHARP |
| MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR |
| MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH |
| MC_8TAP_FN put, regular, REGULAR, REGULAR |
| |
| cglobal put_8tap_16bpc, 4, 9, 16, dst, ds, src, ss, w, h, mx, my |
| %define base r8-put_avx512icl |
| imul mxd, mxm, 0x010101 |
| add mxd, t0d ; 8tap_h, mx, 4tap_h |
| imul myd, mym, 0x010101 |
| add myd, t1d ; 8tap_v, my, 4tap_v |
| lea r8, [put_avx512icl] |
| movifnidn wd, wm |
| movifnidn hd, hm |
| test mxd, 0xf00 |
| jnz .h |
| test myd, 0xf00 |
| jnz .v |
| tzcnt wd, wd |
| movzx wd, word [r8+wq*2+table_offset(put,)] |
| add wq, r8 |
| %if WIN64 |
| pop r8 |
| %endif |
| jmp wq |
| .h_w2: |
| movzx mxd, mxb |
| sub srcq, 2 |
| mova ym2, [spel_h_shuf2a] |
| pmovsxbw xmm4, [base+subpel_filters+mxq*8] |
| pshufd xmm3, xmm4, q1111 |
| pshufd xmm4, xmm4, q2222 |
| .h_w2_loop: |
| movu xm1, [srcq+ssq*0] |
| vinserti32x4 ym1, [srcq+ssq*1], 1 |
| lea srcq, [srcq+ssq*2] |
| mova xmm0, xm8 |
| vpermb ym1, ym2, ym1 |
| vpdpwssd xmm0, xmm3, xm1 |
| vextracti32x4 xm1, ym1, 1 |
| vpdpwssd xmm0, xmm4, xm1 |
| psrad xmm0, 6 |
| packusdw xmm0, xmm0 |
| pminsw xmm0, xm9 |
| movd [dstq+dsq*0], xmm0 |
| pextrd [dstq+dsq*1], xmm0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w2_loop |
| RET |
| .h_w4: |
| movzx mxd, mxb |
| sub srcq, 2 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| vbroadcasti32x4 ym4, [spel_h_shufA] |
| vbroadcasti32x4 ym5, [spel_h_shufB] |
| pshufd xmm0, xmm0, q2211 |
| vpbroadcastq ym6, xmm0 |
| vpermq ym7, ymm0, q1111 |
| .h_w4_loop: |
| movu xm2, [srcq+ssq*0] |
| vinserti32x4 ym2, [srcq+ssq*1], 1 |
| lea srcq, [srcq+ssq*2] |
| mova ym0, ym8 |
| pshufb ym1, ym2, ym4 |
| vpdpwssd ym0, ym6, ym1 |
| pshufb ym2, ym5 |
| vpdpwssd ym0, ym7, ym2 |
| psrad ym0, 6 |
| vextracti32x4 xm1, ym0, 1 |
| packusdw xm0, xm1 |
| pminsw xmm0, xm0, xm9 |
| movq [dstq+dsq*0], xmm0 |
| movhps [dstq+dsq*1], xmm0 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w4_loop |
| RET |
| .h: |
| test myd, 0xf00 |
| jnz .hv |
| mov r7d, r8m |
| vpbroadcastw m9, r8m |
| shr r7d, 11 |
| vpbroadcastd m8, [base+put_8tap_h_rnd+r7*4] |
| cmp wd, 4 |
| je .h_w4 |
| jl .h_w2 |
| shr mxd, 16 |
| sub srcq, 6 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| mova [buf], xmm0 |
| vpbroadcastd m10, xmm0 |
| vpbroadcastd m11, [buf+ 4] |
| vpbroadcastd m12, [buf+ 8] |
| vpbroadcastd m13, [buf+12] |
| sub wd, 16 |
| je .h_w16 |
| jg .h_w32 |
| .h_w8: |
| mova m4, [spel_h_shufA] |
| movu m5, [spel_h_shufB] |
| movu m6, [spel_h_shufC] |
| mova m7, [spel_h_shufD] |
| .h_w8_loop: |
| movu ym2, [srcq+ssq*0] |
| vinserti32x8 m2, [srcq+ssq*1], 1 |
| lea srcq, [srcq+ssq*2] |
| mova m0, m8 |
| vpermb m1, m4, m2 |
| vpdpwssd m0, m10, m1 |
| vpermb m1, m5, m2 |
| vpdpwssd m0, m11, m1 |
| vpermb m1, m6, m2 |
| vpdpwssd m0, m12, m1 |
| vpermb m1, m7, m2 |
| vpdpwssd m0, m13, m1 |
| psrad m0, 6 |
| vextracti32x8 ym1, m0, 1 |
| packusdw ym0, ym1 |
| pminsw ym0, ym9 |
| mova [dstq+dsq*0], xm0 |
| vextracti32x4 [dstq+dsq*1], ym0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w8_loop |
| RET |
| .h_w16: |
| vbroadcasti32x4 m6, [spel_h_shufA] |
| vbroadcasti32x4 m7, [spel_h_shufB] |
| .h_w16_loop: |
| movu ym2, [srcq+ssq*0+ 0] |
| vinserti32x8 m2, [srcq+ssq*1+ 0], 1 |
| movu ym3, [srcq+ssq*0+16] |
| vinserti32x8 m3, [srcq+ssq*1+16], 1 |
| lea srcq, [srcq+ssq*2] |
| mova m0, m8 |
| mova m1, m8 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m10, m4 ; a0 |
| pshufb m4, m3, m6 |
| vpdpwssd m1, m12, m4 ; b2 |
| pshufb m4, m2, m7 |
| vpdpwssd m0, m11, m4 ; a1 |
| pshufb m4, m3, m7 |
| vpdpwssd m1, m13, m4 ; b3 |
| shufpd m2, m3, 0x55 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m12, m4 ; a2 |
| vpdpwssd m1, m10, m4 ; b0 |
| pshufb m2, m7 |
| vpdpwssd m0, m13, m2 ; a3 |
| vpdpwssd m1, m11, m2 ; b1 |
| psrad m0, 6 |
| psrad m1, 6 |
| packusdw m0, m1 |
| pminsw m0, m9 |
| mova [dstq+dsq*0], ym0 |
| vextracti32x8 [dstq+dsq*1], m0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .h_w16_loop |
| RET |
| .h_w32: |
| lea srcq, [srcq+wq*2] |
| vbroadcasti32x4 m6, [spel_h_shufA] |
| lea dstq, [dstq+wq*2] |
| vbroadcasti32x4 m7, [spel_h_shufB] |
| neg wq |
| .h_w32_loop0: |
| mov r6, wq |
| .h_w32_loop: |
| movu m2, [srcq+r6*2+ 0] |
| movu m3, [srcq+r6*2+ 8] |
| mova m0, m8 |
| mova m1, m8 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m10, m4 ; a0 |
| pshufb m4, m3, m6 |
| vpdpwssd m1, m10, m4 ; b0 |
| vpdpwssd m0, m12, m4 ; a2 |
| movu m4, [srcq+r6*2+16] |
| pshufb m3, m7 |
| vpdpwssd m1, m11, m3 ; b1 |
| vpdpwssd m0, m13, m3 ; a3 |
| pshufb m3, m4, m6 |
| vpdpwssd m1, m12, m3 ; b2 |
| pshufb m2, m7 |
| vpdpwssd m0, m11, m2 ; a1 |
| pshufb m4, m7 |
| vpdpwssd m1, m13, m4 ; b3 |
| psrad m0, 6 |
| psrad m1, 6 |
| packusdw m0, m1 |
| pminsw m0, m9 |
| mova [dstq+r6*2], m0 |
| add r6, 32 |
| jl .h_w32_loop |
| add srcq, ssq |
| add dstq, dsq |
| dec hd |
| jg .h_w32_loop0 |
| RET |
| .v: |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 6 |
| cmovs myd, mxd |
| vpbroadcastd m10, [pd_32] |
| pmovsxbw xmm0, [base+subpel_filters+myq*8] |
| tzcnt r7d, wd |
| vpbroadcastw m11, r8m |
| lea r6, [ssq*3] |
| movzx r7d, word [r8+r7*2+table_offset(put, _8tap_v)] |
| sub srcq, r6 |
| mova [rsp+stack_offset+8], xmm0 |
| vpbroadcastd m12, xmm0 |
| add r7, r8 |
| vpbroadcastd m13, [rsp+stack_offset+12] |
| vpbroadcastd m14, [rsp+stack_offset+16] |
| vpbroadcastd m15, [rsp+stack_offset+20] |
| jmp r7 |
| .v_w2: |
| movd xmm2, [srcq+ssq*0] |
| pinsrd xmm2, [srcq+ssq*1], 1 |
| pinsrd xmm2, [srcq+ssq*2], 2 |
| add srcq, r6 |
| pinsrd xmm2, [srcq+ssq*0], 3 ; 0 1 2 3 |
| movd xmm3, [srcq+ssq*1] |
| vpbroadcastd xmm1, [srcq+ssq*2] |
| add srcq, r6 |
| vpbroadcastd xmm0, [srcq+ssq*0] |
| vpblendd xmm3, xmm1, 0x02 ; 4 5 |
| vpblendd xmm1, xmm0, 0x02 ; 5 6 |
| palignr xmm4, xmm3, xmm2, 4 ; 1 2 3 4 |
| punpcklwd xmm3, xmm1 ; 45 56 |
| punpcklwd xmm1, xmm2, xmm4 ; 01 12 |
| punpckhwd xmm2, xmm4 ; 23 34 |
| .v_w2_loop: |
| vpbroadcastd xmm4, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mova xmm5, xm10 |
| vpdpwssd xmm5, xm12, xmm1 ; a0 b0 |
| mova xmm1, xmm2 |
| vpdpwssd xmm5, xm13, xmm2 ; a1 b1 |
| mova xmm2, xmm3 |
| vpdpwssd xmm5, xm14, xmm3 ; a2 b2 |
| vpblendd xmm3, xmm0, xmm4, 0x02 ; 6 7 |
| vpbroadcastd xmm0, [srcq+ssq*0] |
| vpblendd xmm4, xmm0, 0x02 ; 7 8 |
| punpcklwd xmm3, xmm4 ; 67 78 |
| vpdpwssd xmm5, xm15, xmm3 ; a3 b3 |
| psrad xmm5, 6 |
| packusdw xmm5, xmm5 |
| pminsw xmm5, xm11 |
| movd [dstq+dsq*0], xmm5 |
| pextrd [dstq+dsq*1], xmm5, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w2_loop |
| RET |
| .v_w4: |
| movq xmm1, [srcq+ssq*0] |
| vpbroadcastq ymm0, [srcq+ssq*1] |
| vpbroadcastq ymm2, [srcq+ssq*2] |
| add srcq, r6 |
| vpbroadcastq ymm4, [srcq+ssq*0] |
| vpbroadcastq ymm3, [srcq+ssq*1] |
| vpbroadcastq ymm5, [srcq+ssq*2] |
| add srcq, r6 |
| vpblendd ymm1, ymm0, 0x30 |
| vpblendd ymm0, ymm2, 0x30 |
| punpcklwd ymm1, ymm0 ; 01 12 |
| vpbroadcastq ymm0, [srcq+ssq*0] |
| vpblendd ymm2, ymm4, 0x30 |
| vpblendd ymm4, ymm3, 0x30 |
| punpcklwd ymm2, ymm4 ; 23 34 |
| vpblendd ymm3, ymm5, 0x30 |
| vpblendd ymm5, ymm0, 0x30 |
| punpcklwd ymm3, ymm5 ; 45 56 |
| .v_w4_loop: |
| vpbroadcastq ymm5, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| mova ymm4, ym10 |
| vpdpwssd ymm4, ym12, ymm1 ; a0 b0 |
| mova ymm1, ymm2 |
| vpdpwssd ymm4, ym13, ymm2 ; a1 b1 |
| mova ymm2, ymm3 |
| vpdpwssd ymm4, ym14, ymm3 ; a2 b2 |
| vpblendd ymm3, ymm0, ymm5, 0x30 |
| vpbroadcastq ymm0, [srcq+ssq*0] |
| vpblendd ymm5, ymm0, 0x30 |
| punpcklwd ymm3, ymm5 ; 67 78 |
| vpdpwssd ymm4, ym15, ymm3 ; a3 b3 |
| psrad ymm4, 6 |
| vextracti128 xmm5, ymm4, 1 |
| packusdw xmm4, xmm5 |
| pminsw xmm4, xm11 |
| movq [dstq+dsq*0], xmm4 |
| movhps [dstq+dsq*1], xmm4 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w4_loop |
| vzeroupper |
| RET |
| .v_w8: |
| vbroadcasti32x4 m2, [srcq+ssq*2] |
| vinserti32x4 m1, m2, [srcq+ssq*0], 0 |
| vinserti32x4 m1, [srcq+ssq*1], 1 ; 0 1 2 |
| add srcq, r6 |
| vinserti32x4 ym2, [srcq+ssq*0], 1 |
| vinserti32x4 m2, [srcq+ssq*1], 2 ; 2 3 4 |
| mova m6, [spel_v_shuf8] |
| movu xm0, [srcq+ssq*1] |
| vinserti32x4 ym0, [srcq+ssq*2], 1 |
| add srcq, r6 |
| vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6 |
| vpermb m1, m6, m1 ; 01 12 |
| vpermb m2, m6, m2 ; 23 34 |
| vpermb m3, m6, m0 ; 45 56 |
| .v_w8_loop: |
| vinserti32x4 m0, [srcq+ssq*1], 3 |
| lea srcq, [srcq+ssq*2] |
| movu xm5, [srcq+ssq*0] |
| mova m4, m10 |
| vpdpwssd m4, m12, m1 ; a0 b0 |
| mova m1, m2 |
| vshufi32x4 m0, m5, q1032 ; 6 7 8 |
| vpdpwssd m4, m13, m2 ; a1 b1 |
| mova m2, m3 |
| vpdpwssd m4, m14, m3 ; a2 b2 |
| vpermb m3, m6, m0 ; 67 78 |
| vpdpwssd m4, m15, m3 ; a3 b3 |
| psrad m4, 6 |
| vextracti32x8 ym5, m4, 1 |
| packusdw ym4, ym5 |
| pminsw ym4, ym11 |
| mova [dstq+dsq*0], xm4 |
| vextracti32x4 [dstq+dsq*1], ym4, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w8_loop |
| RET |
| .v_w16: |
| vbroadcasti32x8 m1, [srcq+ssq*1] |
| vinserti32x8 m0, m1, [srcq+ssq*0], 0 |
| vinserti32x8 m1, [srcq+ssq*2], 1 |
| mova m8, [spel_v_shuf16] |
| add srcq, r6 |
| movu ym3, [srcq+ssq*0] |
| vinserti32x8 m3, [srcq+ssq*1], 1 |
| movu ym5, [srcq+ssq*2] |
| add srcq, r6 |
| vinserti32x8 m5, [srcq+ssq*0], 1 |
| vpermb m0, m8, m0 ; 01 |
| vpermb m1, m8, m1 ; 12 |
| vpermb m3, m8, m3 ; 34 |
| vpermb m5, m8, m5 ; 56 |
| mova m9, [deint_q_shuf] |
| vpshrdd m2, m1, m3, 16 ; 23 |
| vpshrdd m4, m3, m5, 16 ; 45 |
| .v_w16_loop: |
| mova m6, m10 |
| mova m7, m10 |
| vpdpwssd m6, m12, m0 ; a0 |
| mova m0, m2 |
| vpdpwssd m7, m12, m1 ; b0 |
| mova m1, m3 |
| vpdpwssd m6, m13, m2 ; a1 |
| mova m2, m4 |
| vpdpwssd m7, m13, m3 ; b1 |
| mova m3, m5 |
| vpdpwssd m6, m14, m4 ; a2 |
| mova m4, m5 |
| vpdpwssd m7, m14, m5 ; b2 |
| movu ym5, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x8 m5, [srcq+ssq*0], 1 |
| vpermb m5, m8, m5 ; 78 |
| vpshrdd m4, m5, 16 ; 67 |
| vpdpwssd m6, m15, m4 ; a3 |
| vpdpwssd m7, m15, m5 ; b3 |
| psrad m6, 6 |
| psrad m7, 6 |
| packusdw m6, m7 |
| pminsw m6, m11 |
| vpermq m6, m9, m6 |
| mova [dstq+dsq*0], ym6 |
| vextracti32x8 [dstq+dsq*1], m6, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w16_loop |
| RET |
| .v_w32: |
| .v_w64: |
| .v_w128: |
| %if WIN64 |
| movaps [rsp+stack_offset+8], xmm6 |
| %endif |
| lea wd, [hq+wq*8-256] |
| mov r7, srcq |
| mov r8, dstq |
| .v_w32_loop0: |
| movu m16, [srcq+ssq*0] |
| movu m17, [srcq+ssq*1] |
| movu m18, [srcq+ssq*2] |
| add srcq, r6 |
| movu m19, [srcq+ssq*0] |
| movu m20, [srcq+ssq*1] |
| movu m21, [srcq+ssq*2] |
| add srcq, r6 |
| movu m22, [srcq+ssq*0] |
| punpcklwd m0, m16, m17 ; 01l |
| punpckhwd m16, m17 ; 01h |
| punpcklwd m1, m17, m18 ; 12l |
| punpckhwd m17, m18 ; 12h |
| punpcklwd m2, m18, m19 ; 23l |
| punpckhwd m18, m19 ; 23h |
| punpcklwd m3, m19, m20 ; 34l |
| punpckhwd m19, m20 ; 34h |
| punpcklwd m4, m20, m21 ; 45l |
| punpckhwd m20, m21 ; 45h |
| punpcklwd m5, m21, m22 ; 56l |
| punpckhwd m21, m22 ; 56h |
| .v_w32_loop: |
| mova m6, m10 |
| vpdpwssd m6, m12, m0 ; a0l |
| mova m8, m10 |
| vpdpwssd m8, m12, m16 ; a0h |
| mova m7, m10 |
| vpdpwssd m7, m12, m1 ; b0l |
| mova m9, m10 |
| vpdpwssd m9, m12, m17 ; b0h |
| mova m0, m2 |
| vpdpwssd m6, m13, m2 ; a1l |
| mova m16, m18 |
| vpdpwssd m8, m13, m18 ; a1h |
| mova m1, m3 |
| vpdpwssd m7, m13, m3 ; b1l |
| mova m17, m19 |
| vpdpwssd m9, m13, m19 ; b1h |
| mova m2, m4 |
| vpdpwssd m6, m14, m4 ; a2l |
| mova m18, m20 |
| vpdpwssd m8, m14, m20 ; a2h |
| mova m3, m5 |
| vpdpwssd m7, m14, m5 ; b2l |
| mova m19, m21 |
| vpdpwssd m9, m14, m21 ; b2h |
| movu m21, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| punpcklwd m4, m22, m21 ; 67l |
| punpckhwd m20, m22, m21 ; 67h |
| movu m22, [srcq+ssq*0] |
| vpdpwssd m6, m15, m4 ; a3l |
| vpdpwssd m8, m15, m20 ; a3h |
| punpcklwd m5, m21, m22 ; 78l |
| punpckhwd m21, m22 ; 78h |
| vpdpwssd m7, m15, m5 ; b3l |
| vpdpwssd m9, m15, m21 ; b3h |
| REPX {psrad x, 6}, m6, m8, m7, m9 |
| packusdw m6, m8 |
| packusdw m7, m9 |
| pminsw m6, m11 |
| pminsw m7, m11 |
| mova [dstq+dsq*0], m6 |
| mova [dstq+dsq*1], m7 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .v_w32_loop |
| add r7, 64 |
| add r8, 64 |
| movzx hd, wb |
| mov srcq, r7 |
| mov dstq, r8 |
| sub wd, 1<<8 |
| jg .v_w32_loop0 |
| %if WIN64 |
| movaps xmm6, [rsp+stack_offset+8] |
| %endif |
| vzeroupper |
| RET |
| .hv: |
| vpbroadcastw m11, r8m |
| cmp wd, 4 |
| jg .hv_w8 |
| movzx mxd, mxb |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 6 |
| cmovs myd, mxd |
| pmovsxbw xmm1, [base+subpel_filters+myq*8] |
| lea r6, [ssq*3] |
| sub srcq, 2 |
| sub srcq, r6 |
| test dword r8m, 0x800 |
| jnz .hv_12bit |
| vpbroadcastd m10, [pd_2176] |
| psllw xmm0, 6 |
| jmp .hv_main |
| .hv_12bit: |
| vpbroadcastd m10, [pd_640] |
| psllw xmm0, 4 |
| psllw xmm1, 2 |
| .hv_main: |
| mova [buf+ 0], xmm0 |
| mova [buf+16], xmm1 |
| vpbroadcastd m8, [buf+ 4] |
| vpbroadcastd m9, [buf+ 8] |
| vpbroadcastd ym12, xmm1 |
| vpbroadcastd ym13, [buf+20] |
| vpbroadcastd ym14, [buf+24] |
| vpbroadcastd ym15, [buf+28] |
| movu xm4, [srcq+ssq*0] |
| vinserti32x4 ym4, [srcq+ssq*1], 1 |
| vinserti32x4 m4, [srcq+ssq*2], 2 |
| add srcq, r6 |
| vinserti32x4 m4, [srcq+ssq*0], 3 ; 0 1 2 3 |
| movu xm0, [srcq+ssq*1] |
| vinserti32x4 ym0, [srcq+ssq*2], 1 |
| add srcq, r6 |
| vinserti32x4 m0, [srcq+ssq*0], 2 ; 4 5 6 |
| cmp wd, 4 |
| je .hv_w4 |
| vbroadcasti32x4 m2, [spel_h_shufA] |
| mova m3, [spel_h_shuf2b] |
| mova ym6, [spel_h_shuf2a] |
| mova xm7, [spel_shuf2] |
| mova m1, m10 |
| pshufb m4, m2 |
| pshufb m0, m2 |
| punpcklqdq m2, m4, m0 |
| vpdpwssd m1, m8, m2 ; 04 15 26 3_ |
| punpckhqdq m4, m0 |
| vpdpwssd m1, m9, m4 |
| vpermb m1, m3, m1 ; 01 12 |
| vextracti32x4 xm2, ym1, 1 ; 23 34 |
| vextracti32x4 xm3, m1, 2 ; 45 56 |
| .hv_w2_loop: |
| movu xm5, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x4 ym5, [srcq+ssq*0], 1 |
| mova xm4, xm10 |
| vpermb ym5, ym6, ym5 |
| pmaddwd xmm0, xm12, xm1 ; a0 b0 |
| vpdpwssd xm4, xm8, xm5 |
| vextracti32x4 xm5, ym5, 1 |
| mova xm1, xm2 |
| vpdpwssd xmm0, xm13, xm2 ; a1 b1 |
| vpdpwssd xm4, xm9, xm5 ; 7 8 |
| mova xm2, xm3 |
| vpdpwssd xmm0, xm14, xm3 ; a2 b2 |
| vpermt2b xm3, xm7, xm4 ; 67 78 |
| vpdpwssd xmm0, xm15, xm3 ; a3 b3 |
| psrad xmm0, 10 |
| packusdw xmm0, xmm0 |
| pminsw xmm0, xm11 |
| movd [dstq+dsq*0], xmm0 |
| pextrd [dstq+dsq*1], xmm0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w2_loop |
| RET |
| .hv_w4: |
| vbroadcasti32x4 m19, [spel_h_shufA] |
| vbroadcasti32x4 m20, [spel_h_shufB] |
| mova ym6, [spel_shuf4a] |
| mova ym7, [spel_shuf4b] |
| mova m2, m10 |
| mova m3, m10 |
| pshufb m1, m4, m19 |
| vpdpwssd m2, m8, m1 |
| pshufb m1, m0, m19 |
| vpdpwssd m3, m8, m1 |
| pshufb m4, m20 |
| vpdpwssd m2, m9, m4 |
| pshufb m0, m20 |
| vpdpwssd m3, m9, m0 |
| vpermb m1, m6, m2 ; 01 12 |
| vshufi32x4 m2, m3, q1032 |
| vpermb m3, m6, m3 ; 45 56 |
| vpermb m2, m6, m2 ; 23 34 |
| .hv_w4_loop: |
| movu xm18, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| vinserti128 ym18, [srcq+ssq*0], 1 |
| mova ym4, ym10 |
| pshufb ym17, ym18, ym19 |
| pmaddwd ym16, ym12, ym1 ; a0 b0 |
| vpdpwssd ym4, ym8, ym17 |
| pshufb ym18, ym20 |
| mova ym1, ym2 |
| vpdpwssd ym16, ym13, ym2 ; a1 b1 |
| vpdpwssd ym4, ym9, ym18 ; 7 8 |
| mova ym2, ym3 |
| vpdpwssd ym16, ym14, ym3 ; a2 b2 |
| vpermt2b ym3, ym7, ym4 ; 67 78 |
| vpdpwssd ym16, ym15, ym3 ; a3 b3 |
| psrad ym16, 10 |
| vextracti128 xm17, ym16, 1 |
| packusdw xm16, xm17 |
| pminsw xm16, xm11 |
| movq [dstq+dsq*0], xm16 |
| movhps [dstq+dsq*1], xm16 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w4_loop |
| vzeroupper |
| RET |
| .hv_w8: |
| shr mxd, 16 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 6 |
| cmovs myd, mxd |
| pmovsxbw xmm1, [base+subpel_filters+myq*8] |
| lea r6, [ssq*3] |
| sub srcq, 6 |
| sub srcq, r6 |
| test dword r8m, 0x800 |
| jnz .hv_w8_12bit |
| vpbroadcastd m10, [pd_2176] |
| psllw xmm0, 6 |
| jmp .hv_w8_main |
| .hv_w8_12bit: |
| vpbroadcastd m10, [pd_640] |
| psllw xmm0, 4 |
| psllw xmm1, 2 |
| .hv_w8_main: |
| mova [buf+ 0], xmm0 |
| mova [buf+16], xmm1 |
| vpbroadcastd m12, xmm0 |
| vpbroadcastd m13, [buf+ 4] |
| vpbroadcastd m14, [buf+ 8] |
| vpbroadcastd m15, [buf+12] |
| vpbroadcastd m16, xmm1 |
| vpbroadcastd m17, [buf+20] |
| vpbroadcastd m18, [buf+24] |
| vpbroadcastd m19, [buf+28] |
| cmp wd, 16 |
| je .hv_w16 |
| jg .hv_w32 |
| mova m5, [spel_h_shufA] |
| movu ym0, [srcq+ssq*0] |
| vinserti32x8 m0, [srcq+ssq*1], 1 ; 0 1 |
| movu ym9, [srcq+ssq*2] |
| add srcq, r6 |
| vinserti32x8 m9, [srcq+ssq*0], 1 ; 2 3 |
| movu ym20, [srcq+ssq*1] |
| vinserti32x8 m20, [srcq+ssq*2], 1 ; 4 5 |
| add srcq, r6 |
| movu ym21, [srcq+ssq*0] ; 6 |
| movu m6, [spel_h_shufB] |
| movu m7, [spel_h_shufC] |
| vpermb m8, m5, m0 |
| mova m1, m10 |
| vpdpwssd m1, m12, m8 ; a0 b0 |
| vpermb m8, m5, m9 |
| mova m2, m10 |
| vpdpwssd m2, m12, m8 ; c0 d0 |
| vpermb m8, m5, m20 |
| mova m3, m10 |
| vpdpwssd m3, m12, m8 ; e0 f0 |
| vpermb m8, m5, m21 |
| mova m4, m10 |
| vpdpwssd m4, m12, m8 ; g0 |
| vpermb m8, m6, m0 |
| vpdpwssd m1, m13, m8 ; a1 b1 |
| vpermb m8, m6, m9 |
| vpdpwssd m2, m13, m8 ; c1 d1 |
| vpermb m8, m6, m20 |
| vpdpwssd m3, m13, m8 ; e1 f1 |
| vpermb m8, m6, m21 |
| vpdpwssd m4, m13, m8 ; g1 |
| vpermb m8, m7, m0 |
| vpdpwssd m1, m14, m8 ; a2 b2 |
| vpermb m8, m7, m9 |
| vpdpwssd m2, m14, m8 ; c2 d2 |
| vpermb m8, m7, m20 |
| vpdpwssd m3, m14, m8 ; e2 f2 |
| vpermb m8, m7, m21 |
| vpdpwssd m4, m14, m8 ; g2 |
| mova m8, [spel_h_shufD] |
| vpermb m0, m8, m0 |
| vpdpwssd m1, m15, m0 ; a3 b3 |
| mova m0, [spel_shuf8a] |
| vpermb m9, m8, m9 |
| vpdpwssd m2, m15, m9 ; c3 d3 |
| mova m9, [spel_shuf8b] |
| vpermb m20, m8, m20 |
| vpdpwssd m3, m15, m20 ; e3 f3 |
| vpermb m21, m8, m21 |
| vpdpwssd m4, m15, m21 ; g3 |
| vpermt2b m1, m0, m2 ; 01 12 |
| vpermt2b m2, m0, m3 ; 23 34 |
| vpermt2b m3, m0, m4 ; 45 56 |
| .hv_w8_loop: |
| movu ym0, [srcq+ssq*1] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x8 m0, [srcq+ssq*0], 1 |
| mova m4, m10 |
| vpermb m21, m5, m0 |
| vpdpwssd m4, m12, m21 ; h0 i0 |
| vpermb m21, m6, m0 |
| pmaddwd m20, m16, m1 ; A0 B0 |
| vpdpwssd m4, m13, m21 ; h1 i1 |
| vpermb m21, m7, m0 |
| mova m1, m2 |
| vpdpwssd m20, m17, m2 ; A1 B1 |
| vpdpwssd m4, m14, m21 ; h2 i2 |
| vpermb m21, m8, m0 |
| mova m2, m3 |
| vpdpwssd m20, m18, m3 ; A2 B2 |
| vpdpwssd m4, m15, m21 ; h3 i3 |
| vpermt2b m3, m9, m4 ; 67 78 |
| vpdpwssd m20, m19, m3 ; A3 B3 |
| psrad m20, 10 |
| vextracti32x8 ym21, m20, 1 |
| packusdw ym20, ym21 |
| pminsw ym20, ym11 |
| mova [dstq+dsq*0], xm20 |
| vextracti128 [dstq+dsq*1], ym20, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w8_loop |
| vzeroupper |
| RET |
| .hv_w16: |
| WIN64_SPILL_XMM 26 |
| vbroadcasti32x8 m5, [srcq+ssq*0+ 8] |
| vinserti32x8 m4, m5, [srcq+ssq*0+ 0], 0 |
| vinserti32x8 m5, [srcq+ssq*0+16], 1 ; 0 |
| movu ym6, [srcq+ssq*1+ 0] |
| movu ym7, [srcq+ssq*1+16] |
| vinserti32x8 m6, [srcq+ssq*2+ 0], 1 |
| vinserti32x8 m7, [srcq+ssq*2+16], 1 ; 1 2 |
| add srcq, r6 |
| movu ym22, [srcq+ssq*0+ 0] |
| movu ym23, [srcq+ssq*0+16] |
| vinserti32x8 m22, [srcq+ssq*1+ 0], 1 |
| vinserti32x8 m23, [srcq+ssq*1+16], 1 ; 3 4 |
| movu ym24, [srcq+ssq*2+ 0] |
| movu ym25, [srcq+ssq*2+16] |
| add srcq, r6 |
| vinserti32x8 m24, [srcq+ssq*0+ 0], 1 |
| vinserti32x8 m25, [srcq+ssq*0+16], 1 ; 5 6 |
| vbroadcasti32x4 m20, [spel_h_shufA] |
| vbroadcasti32x4 m21, [spel_h_shufB] |
| mova m9, [spel_shuf16] |
| pshufb m0, m4, m20 |
| mova m1, m10 |
| vpdpwssd m1, m12, m0 ; a0 |
| pshufb m0, m6, m20 |
| mova m2, m10 |
| vpdpwssd m2, m12, m0 ; b0 |
| pshufb m0, m7, m20 |
| mova m3, m10 |
| vpdpwssd m3, m14, m0 ; c2 |
| pshufb m0, m4, m21 |
| vpdpwssd m1, m13, m0 ; a1 |
| pshufb m0, m6, m21 |
| vpdpwssd m2, m13, m0 ; b1 |
| pshufb m0, m7, m21 |
| vpdpwssd m3, m15, m0 ; c3 |
| pshufb m0, m5, m20 |
| vpdpwssd m1, m14, m0 ; a2 |
| shufpd m6, m7, 0x55 |
| pshufb m7, m6, m20 |
| vpdpwssd m2, m14, m7 ; b2 |
| vpdpwssd m3, m12, m7 ; c0 |
| pshufb m5, m21 |
| vpdpwssd m1, m15, m5 ; a3 |
| pshufb m6, m21 |
| vpdpwssd m2, m15, m6 ; b3 |
| vpdpwssd m3, m13, m6 ; c1 |
| pshufb m0, m22, m20 |
| mova m4, m10 |
| vpdpwssd m4, m12, m0 ; d0 |
| pshufb m0, m23, m20 |
| mova m5, m10 |
| vpdpwssd m5, m14, m0 ; e2 |
| pshufb m0, m24, m20 |
| mova m6, m10 |
| vpdpwssd m6, m12, m0 ; f0 |
| pshufb m0, m25, m20 |
| mova m7, m10 |
| vpdpwssd m7, m14, m0 ; g2 |
| pshufb m0, m22, m21 |
| vpdpwssd m4, m13, m0 ; d1 |
| pshufb m0, m23, m21 |
| vpdpwssd m5, m15, m0 ; e3 |
| pshufb m0, m24, m21 |
| vpdpwssd m6, m13, m0 ; f1 |
| pshufb m0, m25, m21 |
| vpdpwssd m7, m15, m0 ; g3 |
| shufpd m22, m23, 0x55 |
| pshufb m23, m22, m20 |
| vpdpwssd m4, m14, m23 ; d2 |
| vpdpwssd m5, m12, m23 ; e0 |
| shufpd m24, m25, 0x55 |
| pshufb m25, m24, m20 |
| vpdpwssd m6, m14, m25 ; f2 |
| vpdpwssd m7, m12, m25 ; g0 |
| pshufb m22, m21 |
| vpdpwssd m4, m15, m22 ; d3 |
| vpdpwssd m5, m13, m22 ; e1 |
| pshufb m24, m21 |
| vpdpwssd m6, m15, m24 ; f3 |
| vpdpwssd m7, m13, m24 ; g1 |
| pslldq m1, 1 |
| vpermt2b m2, m9, m3 ; 12 |
| vpermt2b m4, m9, m5 ; 34 |
| vpermt2b m6, m9, m7 ; 56 |
| vpshrdd m1, m2, 16 ; 01 |
| vpshrdd m3, m2, m4, 16 ; 23 |
| vpshrdd m5, m4, m6, 16 ; 45 |
| .hv_w16_loop: |
| movu ym24, [srcq+ssq*1+ 0] |
| movu ym25, [srcq+ssq*1+16] |
| lea srcq, [srcq+ssq*2] |
| vinserti32x8 m24, [srcq+ssq*0+ 0], 1 |
| vinserti32x8 m25, [srcq+ssq*0+16], 1 |
| mova m7, m10 |
| mova m8, m10 |
| pshufb m0, m24, m20 |
| vpdpwssd m7, m12, m0 ; h0 |
| pshufb m0, m25, m20 |
| vpdpwssd m8, m14, m0 ; i2 |
| pmaddwd m22, m16, m1 ; A0 |
| mova m1, m3 |
| pmaddwd m23, m16, m2 ; B0 |
| mova m2, m4 |
| pshufb m0, m24, m21 |
| vpdpwssd m7, m13, m0 ; h1 |
| pshufb m0, m25, m21 |
| vpdpwssd m8, m15, m0 ; i3 |
| vpdpwssd m22, m17, m3 ; A1 |
| mova m3, m5 |
| vpdpwssd m23, m17, m4 ; B1 |
| mova m4, m6 |
| shufpd m24, m25, 0x55 |
| pshufb m25, m24, m20 |
| vpdpwssd m7, m14, m25 ; h2 |
| vpdpwssd m8, m12, m25 ; i0 |
| vpdpwssd m22, m18, m5 ; A2 |
| vpdpwssd m23, m18, m6 ; B2 |
| pshufb m24, m21 |
| vpdpwssd m7, m15, m24 ; h3 |
| vpdpwssd m8, m13, m24 ; i1 |
| vpermt2b m7, m9, m8 ; 78 |
| vpshrdd m5, m6, m7, 16 ; 67 |
| vpdpwssd m22, m19, m5 ; A3 |
| vpdpwssd m23, m19, m7 ; B3 |
| mova m6, m7 |
| psrad m22, 10 |
| psrad m23, 10 |
| vshufi32x4 m0, m22, m23, q3232 |
| vinserti32x8 m22, ym23, 1 |
| packusdw m22, m0 |
| pminsw m22, m11 |
| mova [dstq+dsq*0], ym22 |
| vextracti32x8 [dstq+dsq*1], m22, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w16_loop |
| RET |
| .hv_w32: |
| %assign stack_offset stack_offset - stack_size_padded |
| WIN64_SPILL_XMM 32 |
| vbroadcasti32x4 m20, [spel_h_shufA] |
| vbroadcasti32x4 m21, [spel_h_shufB] |
| mova m22, [spel_shuf32] |
| lea wd, [hq+wq*8-256] |
| mov r7, srcq |
| mov r8, dstq |
| .hv_w32_loop0: |
| movu m6, [srcq+ssq*0+ 0] |
| movu m7, [srcq+ssq*0+ 8] |
| movu m8, [srcq+ssq*0+16] |
| mova m0, m10 |
| mova m23, m10 |
| pshufb m9, m6, m20 |
| vpdpwssd m0, m12, m9 ; a0l |
| pshufb m9, m7, m20 |
| vpdpwssd m23, m12, m9 ; a0h |
| vpdpwssd m0, m14, m9 ; a2l |
| pshufb m7, m21 |
| vpdpwssd m23, m13, m7 ; a1h |
| vpdpwssd m0, m15, m7 ; a3l |
| pshufb m7, m8, m20 |
| vpdpwssd m23, m14, m7 ; a2h |
| pshufb m6, m21 |
| vpdpwssd m0, m13, m6 ; a1l |
| pshufb m8, m21 |
| vpdpwssd m23, m15, m8 ; a3h |
| %macro PUT_8TAP_HV_W32 5 ; dst_lo, dst_hi, stride_name, stride[1-2] |
| movu m6, [srcq+%3*%4+ 0] |
| movu m7, [srcq+%3*%4+ 8] |
| movu m8, [srcq+%3*%4+16] |
| %if %4 == 2 |
| add srcq, r6 |
| %endif |
| movu m29, [srcq+%3*%5+ 0] |
| movu m30, [srcq+%3*%5+ 8] |
| movu m31, [srcq+%3*%5+16] |
| %if %5 == 2 |
| add srcq, r6 |
| %endif |
| mova m%1, m10 |
| mova m9, m10 |
| pshufb m%2, m6, m20 |
| vpdpwssd m%1, m12, m%2 ; x0l |
| pshufb m%2, m29, m20 |
| vpdpwssd m9, m12, m%2 ; y0l |
| pshufb m6, m21 |
| vpdpwssd m%1, m13, m6 ; x1l |
| pshufb m29, m21 |
| vpdpwssd m9, m13, m29 ; y1l |
| pshufb m6, m7, m20 |
| mova m%2, m10 |
| vpdpwssd m%2, m12, m6 ; x0h |
| pshufb m29, m30, m20 |
| vpdpwssd m%1, m14, m6 ; y2l |
| mova m6, m10 |
| vpdpwssd m6, m12, m29 ; x0h |
| pshufb m7, m21 |
| vpdpwssd m9, m14, m29 ; y2l |
| pshufb m30, m21 |
| vpdpwssd m%2, m13, m7 ; x1h |
| vpdpwssd m%1, m15, m7 ; x3l |
| pshufb m7, m8, m20 |
| vpdpwssd m6, m13, m30 ; y1h |
| vpdpwssd m9, m15, m30 ; y3l |
| pshufb m30, m31, m20 |
| vpdpwssd m%2, m14, m7 ; x2h |
| pshufb m8, m21 |
| vpdpwssd m6, m14, m30 ; y2h |
| pshufb m31, m21 |
| vpdpwssd m%2, m15, m8 ; x3h |
| vpdpwssd m6, m15, m31 ; y3h |
| %if %1 == 1 |
| vpermt2b m0, m22, m%1 ; 01l |
| vpermt2b m23, m22, m%2 ; 01h |
| %endif |
| vpermt2b m%1, m22, m9 ; xyl |
| vpermt2b m%2, m22, m6 ; xyh |
| %endmacro |
| PUT_8TAP_HV_W32 1, 24, ssq, 1, 2 ; 12 |
| PUT_8TAP_HV_W32 3, 26, ssq, 0, 1 ; 34 |
| PUT_8TAP_HV_W32 5, 28, ssq, 2, 0 ; 56 |
| vpshrdd m2, m1, m3, 16 ; 23l |
| vpshrdd m25, m24, m26, 16 ; 23h |
| vpshrdd m4, m3, m5, 16 ; 45l |
| vpshrdd m27, m26, m28, 16 ; 45h |
| .hv_w32_loop: |
| movu m7, [srcq+ssq*1+ 0] |
| movu m9, [srcq+ssq*2+ 0] |
| movu m6, [srcq+ssq*1+ 8] |
| movu m8, [srcq+ssq*2+ 8] |
| mova m29, m10 |
| mova m31, m10 |
| pshufb m30, m7, m20 |
| vpdpwssd m29, m12, m30 ; h0l |
| pshufb m30, m9, m20 |
| vpdpwssd m31, m12, m30 ; i0l |
| pshufb m7, m21 |
| vpdpwssd m29, m13, m7 ; h1l |
| pshufb m9, m21 |
| vpdpwssd m31, m13, m9 ; i1l |
| pshufb m7, m6, m20 |
| vpdpwssd m29, m14, m7 ; h2l |
| pshufb m9, m8, m20 |
| vpdpwssd m31, m14, m9 ; i2l |
| pshufb m6, m21 |
| vpdpwssd m29, m15, m6 ; h3l |
| pshufb m8, m21 |
| vpdpwssd m31, m15, m8 ; i3l |
| mova m30, m10 |
| vpdpwssd m30, m12, m7 ; h0h |
| movu m7, [srcq+ssq*1+16] |
| lea srcq, [srcq+ssq*2] |
| vpermt2b m29, m22, m31 ; 78l |
| mova m31, m10 |
| vpdpwssd m31, m12, m9 ; i0h |
| movu m9, [srcq+ssq*0+16] |
| vpdpwssd m30, m13, m6 ; h1h |
| pshufb m6, m7, m20 |
| vpdpwssd m31, m13, m8 ; i1h |
| pshufb m8, m9, m20 |
| vpdpwssd m30, m14, m6 ; h2h |
| pmaddwd m6, m16, m0 ; A0l |
| pshufb m7, m21 |
| vpdpwssd m31, m14, m8 ; i2h |
| pmaddwd m8, m16, m23 ; A0h |
| pshufb m9, m21 |
| vpdpwssd m30, m15, m7 ; h3h |
| pmaddwd m7, m16, m1 ; B0l |
| vpdpwssd m31, m15, m9 ; i3h |
| pmaddwd m9, m16, m24 ; B0h |
| mova m0, m2 |
| vpdpwssd m6, m17, m2 ; A1l |
| mova m23, m25 |
| vpdpwssd m8, m17, m25 ; A1h |
| mova m1, m3 |
| vpdpwssd m7, m17, m3 ; B1l |
| mova m24, m26 |
| vpdpwssd m9, m17, m26 ; B1h |
| vpermt2b m30, m22, m31 ; 78h |
| vpdpwssd m6, m18, m4 ; A2l |
| mova m2, m4 |
| vpdpwssd m8, m18, m27 ; A2h |
| mova m25, m27 |
| vpdpwssd m7, m18, m5 ; B2l |
| mova m3, m5 |
| vpdpwssd m9, m18, m28 ; B2h |
| mova m26, m28 |
| vpshrdd m4, m5, m29, 16 ; 67l |
| vpdpwssd m6, m19, m4 ; A3l |
| vpshrdd m27, m28, m30, 16 ; 67h |
| vpdpwssd m8, m19, m27 ; A3h |
| mova m5, m29 |
| vpdpwssd m7, m19, m29 ; B3l |
| mova m28, m30 |
| vpdpwssd m9, m19, m30 ; B3h |
| REPX {psrad x, 10}, m6, m8, m7, m9 |
| packusdw m6, m8 |
| packusdw m7, m9 |
| pminsw m6, m11 |
| pminsw m7, m11 |
| mova [dstq+dsq*0], m6 |
| mova [dstq+dsq*1], m7 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .hv_w32_loop |
| add r7, 64 |
| add r8, 64 |
| movzx hd, wb |
| mov srcq, r7 |
| mov dstq, r8 |
| sub wd, 1<<8 |
| jg .hv_w32_loop0 |
| RET |
| |
| %if WIN64 |
| DECLARE_REG_TMP 6, 4 |
| %else |
| DECLARE_REG_TMP 6, 7 |
| %endif |
| |
| MC_8TAP_FN prep, sharp, SHARP, SHARP |
| MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH |
| MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP |
| MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH |
| MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR |
| MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP |
| MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR |
| MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH |
| MC_8TAP_FN prep, regular, REGULAR, REGULAR |
| |
| cglobal prep_8tap_16bpc, 3, 8, 16, tmp, src, stride, w, h, mx, my, stride3 |
| %define base r7-prep_avx512icl |
| imul mxd, mxm, 0x010101 |
| add mxd, t0d ; 8tap_h, mx, 4tap_h |
| imul myd, mym, 0x010101 |
| add myd, t1d ; 8tap_v, my, 4tap_v |
| lea r7, [prep_avx512icl] |
| mov wd, wm |
| movifnidn hd, hm |
| test mxd, 0xf00 |
| jnz .h |
| test myd, 0xf00 |
| jnz .v |
| tzcnt wd, wd |
| mov r5d, r7m ; bitdepth_max |
| vpbroadcastd m5, [pw_8192] |
| movzx wd, word [r7+wq*2+table_offset(prep,)] |
| shr r5d, 11 |
| vpbroadcastd m4, [r7-prep_avx512icl+prep_mul+r5*4] |
| add wq, r7 |
| lea r6, [strideq*3] |
| %if WIN64 |
| pop r7 |
| %endif |
| jmp wq |
| .h_w4: |
| movzx mxd, mxb |
| sub srcq, 2 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| mov r5d, r7m |
| vbroadcasti32x4 m4, [spel_h_shufA] |
| vbroadcasti32x4 m5, [spel_h_shufB] |
| shr r5d, 11 |
| mova ym9, [prep_endA] |
| psllw xmm0, [base+prep_hv_shift+r5*8] |
| mova [tmpq], xmm0 |
| vpbroadcastd m6, [tmpq+4] |
| vpbroadcastd m7, [tmpq+8] |
| .h_w4_loop: |
| movu xm2, [srcq+strideq*0] |
| vinserti32x4 ym2, [srcq+strideq*1], 1 |
| vinserti32x4 m2, [srcq+strideq*2], 2 |
| vinserti32x4 m2, [srcq+r6 ], 3 |
| lea srcq, [srcq+strideq*4] |
| mova m0, m10 |
| pshufb m1, m2, m4 |
| vpdpwssd m0, m6, m1 |
| pshufb m2, m5 |
| vpdpwssd m0, m7, m2 |
| vpermb m0, m9, m0 |
| mova [tmpq], ym0 |
| add tmpq, 32 |
| sub hd, 4 |
| jg .h_w4_loop |
| RET |
| .h: |
| test myd, 0xf00 |
| jnz .hv |
| vpbroadcastd m10, [prep_8tap_rnd] |
| lea r6, [strideq*3] |
| cmp wd, 4 |
| je .h_w4 |
| shr mxd, 16 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| mov r5d, r7m |
| sub srcq, 6 |
| shr r5d, 11 |
| psllw xmm0, [base+prep_hv_shift+r5*8] |
| mova [tmpq], xmm0 |
| vpbroadcastd m12, xmm0 |
| vpbroadcastd m13, [tmpq+ 4] |
| vpbroadcastd m14, [tmpq+ 8] |
| vpbroadcastd m15, [tmpq+12] |
| cmp wd, 16 |
| je .h_w16 |
| jg .h_w32 |
| .h_w8: |
| mova m6, [spel_h_shufA] |
| movu m7, [spel_h_shufB] |
| movu m8, [spel_h_shufC] |
| mova m9, [spel_h_shufD] |
| mova m11, [prep_endB] |
| .h_w8_loop: |
| movu ym4, [srcq+strideq*0] |
| vinserti32x8 m4, [srcq+strideq*1], 1 |
| movu ym5, [srcq+strideq*2] |
| vinserti32x8 m5, [srcq+r6 ], 1 |
| lea srcq, [srcq+strideq*4] |
| mova m0, m10 |
| mova m1, m10 |
| vpermb m2, m6, m4 |
| vpermb m3, m6, m5 |
| vpdpwssd m0, m12, m2 |
| vpdpwssd m1, m12, m3 |
| vpermb m2, m7, m4 |
| vpermb m3, m7, m5 |
| vpdpwssd m0, m13, m2 |
| vpdpwssd m1, m13, m3 |
| vpermb m2, m8, m4 |
| vpermb m3, m8, m5 |
| vpdpwssd m0, m14, m2 |
| vpdpwssd m1, m14, m3 |
| vpermb m2, m9, m4 |
| vpermb m3, m9, m5 |
| vpdpwssd m0, m15, m2 |
| vpdpwssd m1, m15, m3 |
| vpermt2b m0, m11, m1 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| sub hd, 4 |
| jg .h_w8_loop |
| RET |
| .h_w16: |
| vbroadcasti32x4 m6, [spel_h_shufA] |
| vbroadcasti32x4 m7, [spel_h_shufB] |
| mova m11, [prep_endC] |
| .h_w16_loop: |
| movu ym2, [srcq+strideq*0+ 0] |
| vinserti32x8 m2, [srcq+strideq*1+ 0], 1 |
| movu ym3, [srcq+strideq*0+16] |
| vinserti32x8 m3, [srcq+strideq*1+16], 1 |
| lea srcq, [srcq+strideq*2] |
| mova m0, m10 |
| mova m1, m10 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m12, m4 ; a0 |
| pshufb m4, m3, m6 |
| vpdpwssd m1, m14, m4 ; b2 |
| pshufb m4, m2, m7 |
| vpdpwssd m0, m13, m4 ; a1 |
| pshufb m4, m3, m7 |
| vpdpwssd m1, m15, m4 ; b3 |
| shufpd m2, m3, 0x55 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m14, m4 ; a2 |
| vpdpwssd m1, m12, m4 ; b0 |
| pshufb m2, m7 |
| vpdpwssd m0, m15, m2 ; a3 |
| vpdpwssd m1, m13, m2 ; b1 |
| vpermt2b m0, m11, m1 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| sub hd, 2 |
| jg .h_w16_loop |
| RET |
| .h_w32: |
| vbroadcasti32x4 m6, [spel_h_shufA] |
| lea srcq, [srcq+wq*2] |
| vbroadcasti32x4 m7, [spel_h_shufB] |
| neg wq |
| mova m11, [prep_endC] |
| .h_w32_loop0: |
| mov r6, wq |
| .h_w32_loop: |
| movu m2, [srcq+r6*2+ 0] |
| movu m3, [srcq+r6*2+ 8] |
| mova m0, m10 |
| mova m1, m10 |
| pshufb m4, m2, m6 |
| vpdpwssd m0, m12, m4 ; a0 |
| pshufb m4, m3, m6 |
| vpdpwssd m1, m12, m4 ; b0 |
| vpdpwssd m0, m14, m4 ; a2 |
| movu m4, [srcq+r6*2+16] |
| pshufb m3, m7 |
| vpdpwssd m1, m13, m3 ; b1 |
| vpdpwssd m0, m15, m3 ; a3 |
| pshufb m3, m4, m6 |
| vpdpwssd m1, m14, m3 ; b2 |
| pshufb m2, m7 |
| vpdpwssd m0, m13, m2 ; a1 |
| pshufb m4, m7 |
| vpdpwssd m1, m15, m4 ; b3 |
| vpermt2b m0, m11, m1 |
| mova [tmpq], m0 |
| add tmpq, 64 |
| add r6, 32 |
| jl .h_w32_loop |
| add srcq, strideq |
| dec hd |
| jg .h_w32_loop0 |
| RET |
| .v: |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 4 |
| cmove myd, mxd |
| mov r5d, r7m |
| vpbroadcastd m10, [prep_8tap_rnd] |
| pmovsxbw xmm0, [base+subpel_filters+myq*8] |
| tzcnt r6d, wd |
| shr r5d, 11 |
| movzx r6d, word [r7+r6*2+table_offset(prep, _8tap_v)] |
| psllw xmm0, [base+prep_hv_shift+r5*8] |
| add r7, r6 |
| lea r6, [strideq*3] |
| sub srcq, r6 |
| mova [tmpq], xmm0 |
| vpbroadcastd m12, xmm0 |
| vpbroadcastd m13, [tmpq+ 4] |
| vpbroadcastd m14, [tmpq+ 8] |
| vpbroadcastd m15, [tmpq+12] |
| jmp r7 |
| .v_w4: |
| movq xmm1, [srcq+strideq*0] |
| vpbroadcastq ymm0, [srcq+strideq*1] |
| vpbroadcastq ymm2, [srcq+strideq*2] |
| add srcq, r6 |
| vpbroadcastq ymm4, [srcq+strideq*0] |
| vpbroadcastq ymm3, [srcq+strideq*1] |
| vpbroadcastq ymm5, [srcq+strideq*2] |
| mova xm11, [prep_endA] |
| add srcq, r6 |
| vpblendd ymm1, ymm0, 0x30 |
| vpblendd ymm0, ymm2, 0x30 |
| punpcklwd ymm1, ymm0 ; 01 12 |
| vpbroadcastq ymm0, [srcq+strideq*0] |
| vpblendd ymm2, ymm4, 0x30 |
| vpblendd ymm4, ymm3, 0x30 |
| punpcklwd ymm2, ymm4 ; 23 34 |
| vpblendd ymm3, ymm5, 0x30 |
| vpblendd ymm5, ymm0, 0x30 |
| punpcklwd ymm3, ymm5 ; 45 56 |
| .v_w4_loop: |
| vpbroadcastq ymm5, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| mova ymm4, ym10 |
| vpdpwssd ymm4, ym12, ymm1 ; a0 b0 |
| mova ymm1, ymm2 |
| vpdpwssd ymm4, ym13, ymm2 ; a1 b1 |
| mova ymm2, ymm3 |
| vpdpwssd ymm4, ym14, ymm3 ; a2 b2 |
| vpblendd ymm3, ymm0, ymm5, 0x30 |
| vpbroadcastq ymm0, [srcq+strideq*0] |
| vpblendd ymm5, ymm0, 0x30 |
| punpcklwd ymm3, ymm5 ; 67 78 |
| vpdpwssd ymm4, ym15, ymm3 ; a3 b3 |
| vpermb ymm4, ym11, ymm4 |
| mova [tmpq], xmm4 |
| add tmpq, 16 |
| sub hd, 2 |
| jg .v_w4_loop |
| vzeroupper |
| RET |
| .v_w8: |
| vbroadcasti32x4 m2, [srcq+strideq*2] |
| vinserti32x4 m1, m2, [srcq+strideq*0], 0 |
| vinserti32x4 m1, [srcq+strideq*1], 1 ; 0 1 2 |
| add srcq, r6 |
| vinserti32x4 ym2, [srcq+strideq*0], 1 |
| vinserti32x4 m2, [srcq+strideq*1], 2 ; 2 3 4 |
| mova m6, [spel_v_shuf8] |
| movu xm0, [srcq+strideq*1] |
| vinserti32x4 ym0, [srcq+strideq*2], 1 |
| add srcq, r6 |
| vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6 |
| mova ym11, [prep_endB] |
| vpermb m1, m6, m1 ; 01 12 |
| vpermb m2, m6, m2 ; 23 34 |
| vpermb m3, m6, m0 ; 45 56 |
| .v_w8_loop: |
| vinserti32x4 m0, [srcq+strideq*1], 3 |
| lea srcq, [srcq+strideq*2] |
| movu xm5, [srcq+strideq*0] |
| mova m4, m10 |
| vpdpwssd m4, m12, m1 ; a0 b0 |
| mova m1, m2 |
| vshufi32x4 m0, m5, q1032 ; 6 7 8 |
| vpdpwssd m4, m13, m2 ; a1 b1 |
| mova m2, m3 |
| vpdpwssd m4, m14, m3 ; a2 b2 |
| vpermb m3, m6, m0 ; 67 78 |
| vpdpwssd m4, m15, m3 ; a3 b3 |
| vpermb m4, m11, m4 |
| mova [tmpq], ym4 |
| add tmpq, 32 |
| sub hd, 2 |
| jg .v_w8_loop |
| RET |
| .v_w16: |
| vbroadcasti32x8 m1, [srcq+strideq*1] |
| vinserti32x8 m0, m1, [srcq+strideq*0], 0 |
| vinserti32x8 m1, [srcq+strideq*2], 1 |
| mova m8, [spel_v_shuf16] |
| add srcq, r6 |
| movu ym3, [srcq+strideq*0] |
| vinserti32x8 m3, [srcq+strideq*1], 1 |
| movu ym5, [srcq+strideq*2] |
| add srcq, r6 |
| vinserti32x8 m5, [srcq+strideq*0], 1 |
| mova m11, [prep_endA] |
| vpermb m0, m8, m0 ; 01 |
| vpermb m1, m8, m1 ; 12 |
| vpermb m3, m8, m3 ; 34 |
| vpermb m5, m8, m5 ; 56 |
| vpshrdd m2, m1, m3, 16 ; 23 |
| vpshrdd m4, m3, m5, 16 ; 45 |
| .v_w16_loop: |
| mova m6, m10 |
| mova m7, m10 |
| vpdpwssd m6, m12, m0 ; a0 |
| mova m0, m2 |
| vpdpwssd m7, m12, m1 ; b0 |
| mova m1, m3 |
| vpdpwssd m6, m13, m2 ; a1 |
| mova m2, m4 |
| vpdpwssd m7, m13, m3 ; b1 |
| mova m3, m5 |
| vpdpwssd m6, m14, m4 ; a2 |
| mova m4, m5 |
| vpdpwssd m7, m14, m5 ; b2 |
| movu ym5, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| vinserti32x8 m5, [srcq+strideq*0], 1 |
| vpermb m5, m8, m5 ; 78 |
| vpshrdd m4, m5, 16 ; 67 |
| vpdpwssd m6, m15, m4 ; a3 |
| vpdpwssd m7, m15, m5 ; b3 |
| vpermt2b m6, m11, m7 |
| mova [tmpq], m6 |
| add tmpq, 64 |
| sub hd, 2 |
| jg .v_w16_loop |
| RET |
| .v_w32: |
| .v_w64: |
| .v_w128: |
| %if WIN64 |
| PUSH r8 |
| movaps [rsp+stack_offset+8], xmm6 |
| %endif |
| lea r5, [hq+wq*8-256] |
| mov r7, srcq |
| mov r8, tmpq |
| .v_w32_loop0: |
| movu m16, [srcq+strideq*0] |
| movu m17, [srcq+strideq*1] |
| movu m18, [srcq+strideq*2] |
| add srcq, r6 |
| movu m19, [srcq+strideq*0] |
| movu m20, [srcq+strideq*1] |
| movu m21, [srcq+strideq*2] |
| add srcq, r6 |
| movu m22, [srcq+strideq*0] |
| mova m11, [prep_endC] |
| punpcklwd m0, m16, m17 ; 01l |
| punpckhwd m16, m17 ; 01h |
| punpcklwd m1, m17, m18 ; 12l |
| punpckhwd m17, m18 ; 12h |
| punpcklwd m2, m18, m19 ; 23l |
| punpckhwd m18, m19 ; 23h |
| punpcklwd m3, m19, m20 ; 34l |
| punpckhwd m19, m20 ; 34h |
| punpcklwd m4, m20, m21 ; 45l |
| punpckhwd m20, m21 ; 45h |
| punpcklwd m5, m21, m22 ; 56l |
| punpckhwd m21, m22 ; 56h |
| .v_w32_loop: |
| mova m6, m10 |
| vpdpwssd m6, m12, m0 ; a0l |
| mova m8, m10 |
| vpdpwssd m8, m12, m16 ; a0h |
| mova m7, m10 |
| vpdpwssd m7, m12, m1 ; b0l |
| mova m9, m10 |
| vpdpwssd m9, m12, m17 ; b0h |
| mova m0, m2 |
| vpdpwssd m6, m13, m2 ; a1l |
| mova m16, m18 |
| vpdpwssd m8, m13, m18 ; a1h |
| mova m1, m3 |
| vpdpwssd m7, m13, m3 ; b1l |
| mova m17, m19 |
| vpdpwssd m9, m13, m19 ; b1h |
| mova m2, m4 |
| vpdpwssd m6, m14, m4 ; a2l |
| mova m18, m20 |
| vpdpwssd m8, m14, m20 ; a2h |
| mova m3, m5 |
| vpdpwssd m7, m14, m5 ; b2l |
| mova m19, m21 |
| vpdpwssd m9, m14, m21 ; b2h |
| movu m21, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| punpcklwd m4, m22, m21 ; 67l |
| punpckhwd m20, m22, m21 ; 67h |
| movu m22, [srcq+strideq*0] |
| vpdpwssd m6, m15, m4 ; a3l |
| vpdpwssd m8, m15, m20 ; a3h |
| punpcklwd m5, m21, m22 ; 78l |
| punpckhwd m21, m22 ; 78h |
| vpdpwssd m7, m15, m5 ; b3l |
| vpdpwssd m9, m15, m21 ; b3h |
| vpermt2b m6, m11, m8 |
| vpermt2b m7, m11, m9 |
| mova [tmpq+wq*0], m6 |
| mova [tmpq+wq*2], m7 |
| lea tmpq, [tmpq+wq*4] |
| sub hd, 2 |
| jg .v_w32_loop |
| add r7, 64 |
| add r8, 64 |
| movzx hd, r5b |
| mov srcq, r7 |
| mov tmpq, r8 |
| sub r5d, 1<<8 |
| jg .v_w32_loop0 |
| %if WIN64 |
| movaps xmm6, [rsp+stack_offset+8] |
| POP r8 |
| %endif |
| vzeroupper |
| RET |
| .hv: |
| cmp wd, 4 |
| jg .hv_w8 |
| movzx mxd, mxb |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 4 |
| cmove myd, mxd |
| mov r5d, r7m |
| pmovsxbw xmm1, [base+subpel_filters+myq*8] |
| lea r6, [strideq*3] |
| sub srcq, 2 |
| shr r5d, 11 |
| sub srcq, r6 |
| psllw xmm0, [base+prep_hv_shift+r5*8] |
| psllw xmm1, 2 |
| vpbroadcastd m10, [prep_8tap_rnd] |
| vpbroadcastd ym11, [pd_128] |
| mova xm21, [prep_endA] |
| mova [tmpq+ 0], xmm0 |
| mova [tmpq+16], xmm1 |
| vpbroadcastd m8, [tmpq+ 4] |
| vpbroadcastd m9, [tmpq+ 8] |
| vpbroadcastd ym12, xmm1 |
| vpbroadcastd ym13, [tmpq+20] |
| vpbroadcastd ym14, [tmpq+24] |
| vpbroadcastd ym15, [tmpq+28] |
| movu xm4, [srcq+strideq*0] |
| vinserti32x4 ym4, [srcq+strideq*1], 1 |
| vinserti32x4 m4, [srcq+strideq*2], 2 |
| add srcq, r6 |
| vinserti32x4 m4, [srcq+strideq*0], 3 ; 0 1 2 3 |
| movu xm0, [srcq+strideq*1] |
| vinserti32x4 ym0, [srcq+strideq*2], 1 |
| add srcq, r6 |
| vinserti32x4 m0, [srcq+strideq*0], 2 ; 4 5 6 |
| vbroadcasti32x4 m19, [spel_h_shufA] |
| vbroadcasti32x4 m20, [spel_h_shufB] |
| mova ym6, [spel_shuf4a] |
| mova ym7, [spel_shuf4b] |
| mova m2, m10 |
| mova m3, m10 |
| pshufb m1, m4, m19 |
| vpdpwssd m2, m8, m1 |
| pshufb m1, m0, m19 |
| vpdpwssd m3, m8, m1 |
| pshufb m4, m20 |
| vpdpwssd m2, m9, m4 |
| pshufb m0, m20 |
| vpdpwssd m3, m9, m0 |
| vpermb m1, m6, m2 ; 01 12 |
| vshufi32x4 m2, m3, q1032 |
| vpermb m3, m6, m3 ; 45 56 |
| vpermb m2, m6, m2 ; 23 34 |
| .hv_w4_loop: |
| movu xm18, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| vinserti128 ym18, [srcq+strideq*0], 1 |
| mova ym16, ym11 |
| mova ym4, ym10 |
| pshufb ym17, ym18, ym19 |
| vpdpwssd ym16, ym12, ym1 ; a0 b0 |
| vpdpwssd ym4, ym8, ym17 |
| pshufb ym18, ym20 |
| mova ym1, ym2 |
| vpdpwssd ym16, ym13, ym2 ; a1 b1 |
| vpdpwssd ym4, ym9, ym18 ; 7 8 |
| mova ym2, ym3 |
| vpdpwssd ym16, ym14, ym3 ; a2 b2 |
| vpermt2b ym3, ym7, ym4 ; 67 78 |
| vpdpwssd ym16, ym15, ym3 ; a3 b3 |
| vpermb ym16, ym21, ym16 |
| mova [tmpq], xm16 |
| add tmpq, 16 |
| sub hd, 2 |
| jg .hv_w4_loop |
| vzeroupper |
| RET |
| .hv_w8: |
| shr mxd, 16 |
| pmovsxbw xmm0, [base+subpel_filters+mxq*8] |
| movzx mxd, myb |
| shr myd, 16 |
| cmp hd, 6 |
| cmovs myd, mxd |
| mov r5d, r7m |
| pmovsxbw xmm1, [base+subpel_filters+myq*8] |
| lea r6, [strideq*3] |
| sub srcq, 6 |
| shr r5d, 11 |
| sub srcq, r6 |
| vpbroadcastd m10, [prep_8tap_rnd] |
| vpbroadcastd m11, [pd_128] |
| psllw xmm0, [base+prep_hv_shift+r5*8] |
| psllw xmm1, 2 |
| mova [tmpq+ 0], xmm0 |
| mova [tmpq+16], xmm1 |
| vpbroadcastd m12, xmm0 |
| vpbroadcastd m13, [tmpq+ 4] |
| vpbroadcastd m14, [tmpq+ 8] |
| vpbroadcastd m15, [tmpq+12] |
| vpbroadcastd m16, xmm1 |
| vpbroadcastd m17, [tmpq+20] |
| vpbroadcastd m18, [tmpq+24] |
| vpbroadcastd m19, [tmpq+28] |
| cmp wd, 16 |
| je .hv_w16 |
| jg .hv_w32 |
| WIN64_SPILL_XMM 23 |
| mova m5, [spel_h_shufA] |
| movu ym0, [srcq+strideq*0] |
| vinserti32x8 m0, [srcq+strideq*1], 1 ; 0 1 |
| movu ym9, [srcq+strideq*2] |
| add srcq, r6 |
| vinserti32x8 m9, [srcq+strideq*0], 1 ; 2 3 |
| movu ym20, [srcq+strideq*1] |
| vinserti32x8 m20, [srcq+strideq*2], 1 ; 4 5 |
| add srcq, r6 |
| movu ym21, [srcq+strideq*0] ; 6 |
| movu m6, [spel_h_shufB] |
| movu m7, [spel_h_shufC] |
| mova ym22, [prep_endB] |
| vpermb m8, m5, m0 |
| mova m1, m10 |
| vpdpwssd m1, m12, m8 ; a0 b0 |
| vpermb m8, m5, m9 |
| mova m2, m10 |
| vpdpwssd m2, m12, m8 ; c0 d0 |
| vpermb m8, m5, m20 |
| mova m3, m10 |
| vpdpwssd m3, m12, m8 ; e0 f0 |
| vpermb m8, m5, m21 |
| mova m4, m10 |
| vpdpwssd m4, m12, m8 ; g0 |
| vpermb m8, m6, m0 |
| vpdpwssd m1, m13, m8 ; a1 b1 |
| vpermb m8, m6, m9 |
| vpdpwssd m2, m13, m8 ; c1 d1 |
| vpermb m8, m6, m20 |
| vpdpwssd m3, m13, m8 ; e1 f1 |
| vpermb m8, m6, m21 |
| vpdpwssd m4, m13, m8 ; g1 |
| vpermb m8, m7, m0 |
| vpdpwssd m1, m14, m8 ; a2 b2 |
| vpermb m8, m7, m9 |
| vpdpwssd m2, m14, m8 ; c2 d2 |
| vpermb m8, m7, m20 |
| vpdpwssd m3, m14, m8 ; e2 f2 |
| vpermb m8, m7, m21 |
| vpdpwssd m4, m14, m8 ; g2 |
| mova m8, [spel_h_shufD] |
| vpermb m0, m8, m0 |
| vpdpwssd m1, m15, m0 ; a3 b3 |
| mova m0, [spel_shuf8a] |
| vpermb m9, m8, m9 |
| vpdpwssd m2, m15, m9 ; c3 d3 |
| mova m9, [spel_shuf8b] |
| vpermb m20, m8, m20 |
| vpdpwssd m3, m15, m20 ; e3 f3 |
| vpermb m21, m8, m21 |
| vpdpwssd m4, m15, m21 ; g3 |
| vpermt2b m1, m0, m2 ; 01 12 |
| vpermt2b m2, m0, m3 ; 23 34 |
| vpermt2b m3, m0, m4 ; 45 56 |
| .hv_w8_loop: |
| movu ym0, [srcq+strideq*1] |
| lea srcq, [srcq+strideq*2] |
| vinserti32x8 m0, [srcq+strideq*0], 1 |
| mova m4, m10 |
| mova m20, m11 |
| vpermb m21, m5, m0 |
| vpdpwssd m4, m12, m21 ; h0 i0 |
| vpermb m21, m6, m0 |
| vpdpwssd m20, m16, m1 ; A0 B0 |
| vpdpwssd m4, m13, m21 ; h1 i1 |
| vpermb m21, m7, m0 |
| mova m1, m2 |
| vpdpwssd m20, m17, m2 ; A1 B1 |
| vpdpwssd m4, m14, m21 ; h2 i2 |
| vpermb m21, m8, m0 |
| mova m2, m3 |
| vpdpwssd m20, m18, m3 ; A2 B2 |
| vpdpwssd m4, m15, m21 ; h3 i3 |
| vpermt2b m3, m9, m4 ; 67 78 |
| vpdpwssd m20, m19, m3 ; A3 B3 |
| vpermb m20, m22, m20 |
| mova [tmpq], ym20 |
| add tmpq, 32 |
| sub hd, 2 |
| jg .hv_w8_loop |
| RET |
| .hv_w16: |
| %assign stack_offset stack_offset - stack_size_padded |
| WIN64_SPILL_XMM 27 |
| vbroadcasti32x8 m5, [srcq+strideq*0+ 8] |
| vinserti32x8 m4, m5, [srcq+strideq*0+ 0], 0 |
| vinserti32x8 m5, [srcq+strideq*0+16], 1 ; 0 |
| movu ym6, [srcq+strideq*1+ 0] |
| movu ym7, [srcq+strideq*1+16] |
| vinserti32x8 m6, [srcq+strideq*2+ 0], 1 |
| vinserti32x8 m7, [srcq+strideq*2+16], 1 ; 1 2 |
| add srcq, r6 |
| movu ym22, [srcq+strideq*0+ 0] |
| movu ym23, [srcq+strideq*0+16] |
| vinserti32x8 m22, [srcq+strideq*1+ 0], 1 |
| vinserti32x8 m23, [srcq+strideq*1+16], 1 ; 3 4 |
| movu ym24, [srcq+strideq*2+ 0] |
| movu ym25, [srcq+strideq*2+16] |
| add srcq, r6 |
| vinserti32x8 m24, [srcq+strideq*0+ 0], 1 |
| vinserti32x8 m25, [srcq+strideq*0+16], 1 ; 5 6 |
| vbroadcasti32x4 m20, [spel_h_shufA] |
| vbroadcasti32x4 m21, [spel_h_shufB] |
| mova m9, [spel_shuf16] |
| mova m26, [prep_endB] |
| pshufb m0, m4, m20 |
| mova m1, m10 |
| vpdpwssd m1, m12, m0 ; a0 |
| pshufb m0, m6, m20 |
| mova m2, m10 |
| vpdpwssd m2, m12, m0 ; b0 |
| pshufb m0, m7, m20 |
| mova m3, m10 |
| vpdpwssd m3, m14, m0 ; c2 |
| pshufb m0, m4, m21 |
| vpdpwssd m1, m13, m0 ; a1 |
| pshufb m0, m6, m21 |
| vpdpwssd m2, m13, m0 ; b1 |
| pshufb m0, m7, m21 |
| vpdpwssd m3, m15, m0 ; c3 |
| pshufb m0, m5, m20 |
| vpdpwssd m1, m14, m0 ; a2 |
| shufpd m6, m7, 0x55 |
| pshufb m7, m6, m20 |
| vpdpwssd m2, m14, m7 ; b2 |
| vpdpwssd m3, m12, m7 ; c0 |
| pshufb m5, m21 |
| vpdpwssd m1, m15, m5 ; a3 |
| pshufb m6, m21 |
| vpdpwssd m2, m15, m6 ; b3 |
| vpdpwssd m3, m13, m6 ; c1 |
| pshufb m0, m22, m20 |
| mova m4, m10 |
| vpdpwssd m4, m12, m0 ; d0 |
| pshufb m0, m23, m20 |
| mova m5, m10 |
| vpdpwssd m5, m14, m0 ; e2 |
| pshufb m0, m24, m20 |
| mova m6, m10 |
| vpdpwssd m6, m12, m0 ; f0 |
| pshufb m0, m25, m20 |
| mova m7, m10 |
| vpdpwssd m7, m14, m0 ; g2 |
| pshufb m0, m22, m21 |
| vpdpwssd m4, m13, m0 ; d1 |
| pshufb m0, m23, m21 |
| vpdpwssd m5, m15, m0 ; e3 |
| pshufb m0, m24, m21 |
| vpdpwssd m6, m13, m0 ; f1 |
| pshufb m0, m25, m21 |
| vpdpwssd m7, m15, m0 ; g3 |
| shufpd m22, m23, 0x55 |
| pshufb m23, m22, m20 |
| vpdpwssd m4, m14, m23 ; d2 |
| vpdpwssd m5, m12, m23 ; e0 |
| shufpd m24, m25, 0x55 |
| pshufb m25, m24, m20 |
| vpdpwssd m6, m14, m25 ; f2 |
| vpdpwssd m7, m12, m25 ; g0 |
| pshufb m22, m21 |
| vpdpwssd m4, m15, m22 ; d3 |
| vpdpwssd m5, m13, m22 ; e1 |
| pshufb m24, m21 |
| vpdpwssd m6, m15, m24 ; f3 |
| vpdpwssd m7, m13, m24 ; g1 |
| pslldq m1, 1 |
| vpermt2b m2, m9, m3 ; 12 |
| vpermt2b m4, m9, m5 ; 34 |
| vpermt2b m6, m9, m7 ; 56 |
| vpshrdd m1, m2, 16 ; 01 |
| vpshrdd m3, m2, m4, 16 ; 23 |
| vpshrdd m5, m4, m6, 16 ; 45 |
| .hv_w16_loop: |
| movu ym24, [srcq+strideq*1+ 0] |
| movu ym25, [srcq+strideq*1+16] |
| lea srcq, [srcq+strideq*2] |
| vinserti32x8 m24, [srcq+strideq*0+ 0], 1 |
| vinserti32x8 m25, [srcq+strideq*0+16], 1 |
| mova m7, m10 |
| mova m8, m10 |
| pshufb m0, m24, m20 |
| vpdpwssd m7, m12, m0 ; h0 |
| mova m22, m11 |
| pshufb m0, m25, m20 |
| vpdpwssd m8, m14, m0 ; i2 |
| mova m23, m11 |
| vpdpwssd m22, m16, m1 ; A0 |
| mova m1, m3 |
| vpdpwssd m23, m16, m2 ; B0 |
| mova m2, m4 |
| pshufb m0, m24, m21 |
| vpdpwssd m7, m13, m0 ; h1 |
| pshufb m0, m25, m21 |
| vpdpwssd m8, m15, m0 ; i3 |
| vpdpwssd m22, m17, m3 ; A1 |
| mova m3, m5 |
| vpdpwssd m23, m17, m4 ; B1 |
| mova m4, m6 |
| shufpd m24, m25, 0x55 |
| pshufb m25, m24, m20 |
| vpdpwssd m7, m14, m25 ; h2 |
| vpdpwssd m8, m12, m25 ; i0 |
| vpdpwssd m22, m18, m5 ; A2 |
| vpdpwssd m23, m18, m6 ; B2 |
| pshufb m24, m21 |
| vpdpwssd m7, m15, m24 ; h3 |
| vpdpwssd m8, m13, m24 ; i1 |
| vpermt2b m7, m9, m8 ; 78 |
| vpshrdd m5, m6, m7, 16 ; 67 |
| vpdpwssd m22, m19, m5 ; A3 |
| vpdpwssd m23, m19, m7 ; B3 |
| mova m6, m7 |
| vpermt2b m22, m26, m23 |
| mova [tmpq], m22 |
| add tmpq, 64 |
| sub hd, 2 |
| jg .hv_w16_loop |
| RET |
| .hv_w32: |
| %if WIN64 |
| %assign stack_offset stack_offset - stack_size_padded |
| PUSH r8 |
| %assign regs_used regs_used + 1 |
| WIN64_SPILL_XMM 32 |
| %endif |
| vbroadcasti32x4 m20, [spel_h_shufA] |
| vbroadcasti32x4 m21, [spel_h_shufB] |
| mova m22, [spel_shuf32] |
| lea r5d, [hq+wq*8-256] |
| mov r7, srcq |
| mov r8, tmpq |
| .hv_w32_loop0: |
| movu m6, [srcq+strideq*0+ 0] |
| movu m7, [srcq+strideq*0+ 8] |
| movu m8, [srcq+strideq*0+16] |
| mova m0, m10 |
| mova m23, m10 |
| pshufb m9, m6, m20 |
| vpdpwssd m0, m12, m9 ; a0l |
| pshufb m9, m7, m20 |
| vpdpwssd m23, m12, m9 ; a0h |
| vpdpwssd m0, m14, m9 ; a2l |
| pshufb m7, m21 |
| vpdpwssd m23, m13, m7 ; a1h |
| vpdpwssd m0, m15, m7 ; a3l |
| pshufb m7, m8, m20 |
| vpdpwssd m23, m14, m7 ; a2h |
| pshufb m6, m21 |
| vpdpwssd m0, m13, m6 ; a1l |
| pshufb m8, m21 |
| vpdpwssd m23, m15, m8 ; a3h |
| PUT_8TAP_HV_W32 1, 24, strideq, 1, 2 ; 12 |
| PUT_8TAP_HV_W32 3, 26, strideq, 0, 1 ; 34 |
| PUT_8TAP_HV_W32 5, 28, strideq, 2, 0 ; 56 |
| vpshrdd m2, m1, m3, 16 ; 23l |
| vpshrdd m25, m24, m26, 16 ; 23h |
| vpshrdd m4, m3, m5, 16 ; 45l |
| vpshrdd m27, m26, m28, 16 ; 45h |
| .hv_w32_loop: |
| movu m7, [srcq+strideq*1+ 0] |
| movu m9, [srcq+strideq*2+ 0] |
| movu m6, [srcq+strideq*1+ 8] |
| movu m8, [srcq+strideq*2+ 8] |
| mova m29, m10 |
| mova m31, m10 |
| pshufb m30, m7, m20 |
| vpdpwssd m29, m12, m30 ; h0l |
| pshufb m30, m9, m20 |
| vpdpwssd m31, m12, m30 ; i0l |
| pshufb m7, m21 |
| vpdpwssd m29, m13, m7 ; h1l |
| pshufb m9, m21 |
| vpdpwssd m31, m13, m9 ; i1l |
| pshufb m7, m6, m20 |
| vpdpwssd m29, m14, m7 ; h2l |
| pshufb m9, m8, m20 |
| vpdpwssd m31, m14, m9 ; i2l |
| pshufb m6, m21 |
| vpdpwssd m29, m15, m6 ; h3l |
| pshufb m8, m21 |
| vpdpwssd m31, m15, m8 ; i3l |
| mova m30, m10 |
| vpdpwssd m30, m12, m7 ; h0h |
| movu m7, [srcq+strideq*1+16] |
| lea srcq, [srcq+strideq*2] |
| vpermt2b m29, m22, m31 ; 78l |
| mova m31, m10 |
| vpdpwssd m31, m12, m9 ; i0h |
| movu m9, [srcq+strideq*0+16] |
| vpdpwssd m30, m13, m6 ; h1h |
| pshufb m6, m7, m20 |
| vpdpwssd m31, m13, m8 ; i1h |
| pshufb m8, m9, m20 |
| vpdpwssd m30, m14, m6 ; h2h |
| mova m6, m11 |
| vpdpwssd m6, m16, m0 ; A0l |
| pshufb m7, m21 |
| vpdpwssd m31, m14, m8 ; i2h |
| mova m8, m11 |
| vpdpwssd m8, m16, m23 ; A0h |
| pshufb m9, m21 |
| vpdpwssd m30, m15, m7 ; h3h |
| mova m7, m11 |
| vpdpwssd m7, m16, m1 ; B0l |
| vpdpwssd m31, m15, m9 ; i3h |
| mova m9, m11 |
| vpdpwssd m9, m16, m24 ; B0h |
| mova m0, m2 |
| vpdpwssd m6, m17, m2 ; A1l |
| mova m23, m25 |
| vpdpwssd m8, m17, m25 ; A1h |
| mova m1, m3 |
| vpdpwssd m7, m17, m3 ; B1l |
| mova m24, m26 |
| vpdpwssd m9, m17, m26 ; B1h |
| vpermt2b m30, m22, m31 ; 78h |
| mova m31, [prep_endC] |
| vpdpwssd m6, m18, m4 ; A2l |
| mova m2, m4 |
| vpdpwssd m8, m18, m27 ; A2h |
| mova m25, m27 |
| vpdpwssd m7, m18, m5 ; B2l |
| mova m3, m5 |
| vpdpwssd m9, m18, m28 ; B2h |
| mova m26, m28 |
| vpshrdd m4, m5, m29, 16 ; 67l |
| vpdpwssd m6, m19, m4 ; A3l |
| vpshrdd m27, m28, m30, 16 ; 67h |
| vpdpwssd m8, m19, m27 ; A3h |
| mova m5, m29 |
| vpdpwssd m7, m19, m29 ; B3l |
| mova m28, m30 |
| vpdpwssd m9, m19, m30 ; B3h |
| vpermt2b m6, m31, m8 |
| vpermt2b m7, m31, m9 |
| mova [tmpq+wq*0], m6 |
| mova [tmpq+wq*2], m7 |
| lea tmpq, [tmpq+wq*4] |
| sub hd, 2 |
| jg .hv_w32_loop |
| add r7, 64 |
| add r8, 64 |
| movzx hd, r5b |
| mov srcq, r7 |
| mov tmpq, r8 |
| sub r5d, 1<<8 |
| jg .hv_w32_loop0 |
| RET |
| |
| %if WIN64 |
| DECLARE_REG_TMP 5 |
| %else |
| DECLARE_REG_TMP 7 |
| %endif |
| |
| cglobal warp_affine_8x8t_16bpc, 4, 7, 22, tmp, ts |
| %define base r6-pd_0to7 |
| mov t0d, r7m |
| lea r6, [pd_0to7] |
| shr t0d, 11 |
| vpbroadcastd m8, [base+warp_8x8t_rnd_v] |
| vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4] |
| call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main |
| psrad m14, m16, 15 |
| call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 |
| psrad m16, 15 |
| packssdw m14, m16 |
| call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 |
| psrad m15, m16, 15 |
| call mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).main2 |
| add tsq, tsq |
| psrad m16, 15 |
| packssdw m15, m16 |
| jmp mangle(private_prefix %+ _warp_affine_8x8_16bpc_avx512icl).end |
| |
| cglobal warp_affine_8x8_16bpc, 4, 7, 22, dst, ds, src, ss, abcd |
| mov t0d, r7m ; pixel_max |
| lea r6, [pd_0to7] |
| shr t0d, 11 |
| vpbroadcastd m1, [base+warp_8x8_rnd_h+t0*4] |
| vpbroadcastd m8, [base+warp_8x8_rnd_v+t0*4] |
| call .main |
| psrad m14, m16, 13 |
| call .main2 |
| psrad m16, 13 |
| packusdw m14, m16 |
| call .main2 |
| psrad m15, m16, 13 |
| call .main2 |
| vpbroadcastd m0, [base+bidir_shift+t0*4] |
| vpsrlvw m14, m0 |
| psrad m16, 13 |
| packusdw m15, m16 |
| vpsrlvw m15, m0 |
| .end: |
| mova m0, [base+warp8x8_end] |
| vpermb m16, m0, m14 |
| lea r2, [dsq*3] |
| mova [dstq+dsq*0], xm16 |
| vextracti128 [dstq+dsq*1], ym16, 1 |
| vextracti32x4 [dstq+dsq*2], m16, 2 |
| vextracti32x4 [dstq+r2 ], m16, 3 |
| vpermb m16, m0, m15 |
| lea dstq, [dstq+dsq*4] |
| mova [dstq+dsq*0], xm16 |
| vextracti128 [dstq+dsq*1], ym16, 1 |
| vextracti32x4 [dstq+dsq*2], m16, 2 |
| vextracti32x4 [dstq+r2 ], m16, 3 |
| RET |
| .main: |
| vpbroadcastd ym3, [base+pd_512] |
| %if WIN64 |
| mov abcdq, r5mp |
| vpaddd ym18, ym3, r6m {1to8} ; mx |
| %else |
| add r5d, 512 |
| vpbroadcastd ym18, r5d |
| %endif |
| vpaddd ym20, ym3, r7m {1to8} ; my |
| mova ym16, [base+pd_0to7] |
| vpbroadcastd ym19, [abcdq+4*0] ; alpha |
| vpbroadcastd ym21, [abcdq+4*1] ; gamma |
| lea r4, [ssq*3+6] |
| vpdpwssd ym18, ym19, ym16 ; tmx |
| vpdpwssd ym20, ym21, ym16 ; tmy |
| sub srcq, r4 |
| mova m10, [base+warp8x8_permA] |
| lea r4, [mc_warp_filter+64*8] |
| vbroadcasti32x4 m12, [base+warp8x8_permC] |
| kxnorb k1, k1, k1 |
| vbroadcasti32x4 m13, [base+warp8x8_permD] |
| movu ym5, [srcq+0] |
| vinserti32x8 m5, [srcq+8], 1 |
| psrad ym17, ym18, 10 |
| mova m11, [base+warp8x8_permB] |
| kmovb k2, k1 |
| vpgatherdq m3{k1}, [r4+ym17*8] ; filter_x0 |
| psrad ym19, 16 ; beta |
| psrad ym21, 16 ; delta |
| paddd ym18, ym19 |
| vpermb m4, m10, m5 |
| vpbroadcastq m9, [base+warp_shift_h+t0*8] |
| pshufd m3, m3, q3120 |
| paddd m7, m1, m1 |
| pshufb m2, m3, m12 |
| vpdpwssd m1, m4, m2 |
| vpermb m5, m11, m5 |
| vshufi32x4 m4, m5, q1021 |
| pshufb m3, m13 |
| vpdpwssd m1, m4, m3 |
| call .h |
| psllq m2, m1, 32 |
| paddd m1, m2 |
| vpmultishiftqb m1, m9, m1 |
| vpshrdq m1, m0, 48 ; 01 12 |
| call .h |
| vpshrdq m2, m1, m0, 48 ; 23 34 |
| call .h |
| vpshrdq m3, m2, m0, 48 ; 45 56 |
| .main2: |
| call .h |
| psrad ym6, ym20, 10 |
| kmovb k1, k2 |
| paddd ym17, ym20, ym21 ; my += delta |
| vpgatherdq m20{k2}, [r4+ym6*8] ; filter_y0 |
| psrad ym16, ym17, 10 |
| kmovb k2, k1 |
| vpgatherdq m6{k1}, [r4+ym16*8] ; filter_y1 |
| shufps m5, m20, m6, q2020 |
| mova m16, m8 |
| pshufb m4, m5, m12 |
| vpdpwssd m16, m1, m4 ; a0 b0 |
| pshufb m5, m13 |
| mova m1, m2 |
| vpdpwssd m16, m2, m5 ; a1 b1 |
| shufps m6, m20, m6, q3131 |
| paddd ym20, ym17, ym21 |
| pshufb m4, m6, m12 |
| mova m2, m3 |
| vpdpwssd m16, m3, m4 ; a2 b2 |
| vpshrdq m3, m0, 48 ; 67 78 |
| pshufb m6, m13 |
| vpdpwssd m16, m3, m6 ; a3 b3 |
| ret |
| ALIGN function_align |
| .h: |
| movu ym16, [srcq+ssq*1] |
| psrad ym6, ym18, 10 |
| lea srcq, [srcq+ssq*2] |
| vinserti32x8 m5, m16, [srcq+ssq*0], 1 |
| kmovb k1, k2 |
| paddd ym17, ym18, ym19 ; mx += beta |
| vpgatherdq m18{k2}, [r4+ym6*8] ; filter_x1 |
| psrad ym16, ym17, 10 |
| kmovb k2, k1 |
| vpgatherdq m6{k1}, [r4+ym16*8] ; filter_x2 |
| vpermb m4, m10, m5 |
| shufps m16, m18, m6, q2020 |
| shufps m6, m18, m6, q3131 |
| mova m0, m7 |
| pshufb m18, m16, m12 |
| vpdpwssd m0, m4, m18 ; a0 b0 |
| vpermb m5, m11, m5 |
| pshufb m18, m6, m13 |
| vpdpwssd m0, m5, m18 ; a3 b3 |
| paddd ym18, ym17, ym19 |
| vshufi32x4 m17, m4, m5, q1021 |
| pshufb m16, m13 |
| vpdpwssd m0, m17, m16 ; a1 b1 |
| vshufi32x4 m4, m5, q2132 |
| pshufb m6, m12 |
| vpdpwssd m0, m4, m6 ; a2 b2 |
| vpmultishiftqb m0, m9, m0 ; a a b b |
| ret |
| |
| %macro BIDIR_FN 0 |
| call .main |
| lea stride3q, [strideq*3] |
| jmp wq |
| .w4: |
| movq [dstq ], xm0 |
| movhps [dstq+strideq*1], xm0 |
| vextracti32x4 xm2, ym0, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| cmp hd, 8 |
| jl .w4_end |
| vextracti32x4 xm2, m0, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq ], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm0, m0, 3 |
| movq [dstq+strideq*2], xm0 |
| movhps [dstq+stride3q ], xm0 |
| je .w4_end |
| lea dstq, [dstq+strideq*4] |
| movq [dstq ], xm1 |
| movhps [dstq+strideq*1], xm1 |
| vextracti32x4 xm0, ym1, 1 |
| movq [dstq+strideq*2], xm0 |
| movhps [dstq+stride3q ], xm0 |
| vextracti32x4 xm0, m1, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq ], xm0 |
| movhps [dstq+strideq*1], xm0 |
| vextracti32x4 xm1, m1, 3 |
| movq [dstq+strideq*2], xm1 |
| movhps [dstq+stride3q ], xm1 |
| .w4_end: |
| RET |
| .w8_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w8: |
| mova [dstq+strideq*0], xm0 |
| vextracti32x4 [dstq+strideq*1], ym0, 1 |
| vextracti32x4 [dstq+strideq*2], m0, 2 |
| vextracti32x4 [dstq+stride3q ], m0, 3 |
| sub hd, 8 |
| jl .w8_end |
| lea dstq, [dstq+strideq*4] |
| mova [dstq+strideq*0], xm1 |
| vextracti32x4 [dstq+strideq*1], ym1, 1 |
| vextracti32x4 [dstq+strideq*2], m1, 2 |
| vextracti32x4 [dstq+stride3q ], m1, 3 |
| jg .w8_loop |
| .w8_end: |
| RET |
| .w16_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w16: |
| mova [dstq+strideq*0], ym0 |
| vextracti32x8 [dstq+strideq*1], m0, 1 |
| mova [dstq+strideq*2], ym1 |
| vextracti32x8 [dstq+stride3q ], m1, 1 |
| sub hd, 4 |
| jg .w16_loop |
| RET |
| .w32_loop: |
| call .main |
| lea dstq, [dstq+strideq*2] |
| .w32: |
| mova [dstq+strideq*0], m0 |
| mova [dstq+strideq*1], m1 |
| sub hd, 2 |
| jg .w32_loop |
| RET |
| .w64_loop: |
| call .main |
| add dstq, strideq |
| .w64: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| dec hd |
| jg .w64_loop |
| RET |
| .w128_loop: |
| call .main |
| add dstq, strideq |
| .w128: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| call .main |
| mova [dstq+64*2], m0 |
| mova [dstq+64*3], m1 |
| dec hd |
| jg .w128_loop |
| RET |
| %endmacro |
| |
| %if WIN64 |
| DECLARE_REG_TMP 5 |
| %else |
| DECLARE_REG_TMP 7 |
| %endif |
| |
| cglobal avg_16bpc, 4, 7, 4, dst, stride, tmp1, tmp2, w, h, stride3 |
| %define base r6-avg_avx512icl_table |
| lea r6, [avg_avx512icl_table] |
| tzcnt wd, wm |
| mov t0d, r6m ; pixel_max |
| movsxd wq, [r6+wq*4] |
| shr t0d, 11 |
| vpbroadcastd m2, [base+avg_round+t0*4] |
| vpbroadcastd m3, [base+avg_shift+t0*4] |
| movifnidn hd, hm |
| add wq, r6 |
| BIDIR_FN |
| ALIGN function_align |
| .main: |
| mova m0, [tmp1q+64*0] |
| paddsw m0, [tmp2q+64*0] |
| mova m1, [tmp1q+64*1] |
| paddsw m1, [tmp2q+64*1] |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| pmaxsw m0, m2 |
| pmaxsw m1, m2 |
| psubsw m0, m2 |
| psubsw m1, m2 |
| vpsrlvw m0, m3 |
| vpsrlvw m1, m3 |
| ret |
| |
| cglobal w_avg_16bpc, 4, 7, 8, dst, stride, tmp1, tmp2, w, h, stride3 |
| %define base r6-w_avg_avx512icl_table |
| lea r6, [w_avg_avx512icl_table] |
| tzcnt wd, wm |
| mov t0d, r7m ; pixel_max |
| shr t0d, 11 |
| movsxd wq, [r6+wq*4] |
| vpbroadcastd m5, [base+w_avg_round+t0*4] |
| vpbroadcastd m7, [base+bidir_shift+t0*4] |
| add wq, r6 |
| mov r6d, r6m ; weight |
| lea t0d, [r6-16] |
| shl r6d, 16 |
| sub r6d, t0d ; 16-weight, weight |
| movifnidn hd, hm |
| vpbroadcastd m6, r6d |
| BIDIR_FN |
| ALIGN function_align |
| .main: |
| mova m3, [tmp1q+64*0] |
| mova m1, [tmp2q+64*0] |
| mova m0, [tmp1q+64*1] |
| mova m4, [tmp2q+64*1] |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| punpcklwd m2, m1, m3 |
| punpckhwd m1, m3 |
| punpcklwd m3, m4, m0 |
| punpckhwd m4, m0 |
| mova m0, m5 |
| vpdpwssd m0, m6, m2 |
| mova m2, m5 |
| vpdpwssd m2, m6, m1 |
| mova m1, m5 |
| vpdpwssd m1, m6, m3 |
| mova m3, m5 |
| vpdpwssd m3, m6, m4 |
| REPX {psrad x, 2}, m0, m2, m1, m3 |
| packusdw m0, m2 |
| packusdw m1, m3 |
| vpsrlvw m0, m7 |
| vpsrlvw m1, m7 |
| ret |
| |
| cglobal mask_16bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 |
| %define base r7-mask_avx512icl_table |
| lea r7, [mask_avx512icl_table] |
| tzcnt wd, wm |
| mov r6d, r7m ; pixel_max |
| movifnidn hd, hm |
| shr r6d, 11 |
| movsxd wq, [r7+wq*4] |
| vpbroadcastd m8, [base+pw_64] |
| vpbroadcastd m9, [base+mask_round+r6*4] |
| vpbroadcastd m10, [base+bidir_shift+r6*4] |
| mov maskq, maskmp |
| add wq, r7 |
| BIDIR_FN |
| ALIGN function_align |
| .main: |
| pmovzxbw m1, [maskq+32*0] |
| mova m4, [tmp1q+64*0] |
| mova m2, [tmp2q+64*0] |
| pmovzxbw m6, [maskq+32*1] |
| mova m5, [tmp1q+64*1] |
| mova m3, [tmp2q+64*1] |
| add maskq, 32*2 |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| punpcklwd m7, m4, m2 |
| punpckhwd m4, m2 |
| psubw m0, m8, m1 |
| punpcklwd m2, m1, m0 ; m, 64-m |
| punpckhwd m1, m0 |
| mova m0, m9 |
| vpdpwssd m0, m7, m2 |
| mova m2, m9 |
| vpdpwssd m2, m4, m1 ; tmp1 * m + tmp2 * (64-m) |
| punpcklwd m7, m5, m3 |
| punpckhwd m5, m3 |
| psubw m1, m8, m6 |
| punpcklwd m3, m6, m1 |
| punpckhwd m6, m1 |
| mova m1, m9 |
| vpdpwssd m1, m7, m3 |
| mova m3, m9 |
| vpdpwssd m3, m5, m6 |
| REPX {psrad x, 4}, m0, m2, m1, m3 |
| packusdw m0, m2 |
| packusdw m1, m3 |
| vpsrlvw m0, m10 |
| vpsrlvw m1, m10 |
| ret |
| |
| cglobal w_mask_420_16bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 |
| %define base r7-w_mask_420_avx512icl_table |
| lea r7, [w_mask_420_avx512icl_table] |
| tzcnt wd, wm |
| mov r6d, r8m ; pixel_max |
| movifnidn hd, hm |
| shr r6d, 11 |
| movsxd wq, [r7+wq*4] |
| vpbroadcastd m10, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 |
| vpbroadcastd m11, [base+pw_64] |
| vpbroadcastd m12, [base+mask_round+r6*4] |
| vpbroadcastd m13, [base+bidir_shift+r6*4] |
| mov r6d, r7m ; sign |
| vpbroadcastd m14, [base+w_mask_round+r6*4] |
| mova ym15, [w_mask_end42x] |
| mov maskq, maskmp |
| add wq, r7 |
| call .main |
| lea stride3q, [strideq*3] |
| jmp wq |
| .w4: |
| mova m4, [w_mask_shuf4] |
| vpermt2b m2, m4, m3 |
| mova m3, m14 |
| vpdpbusd m3, m2, [pb_64] {1to16} |
| vpermb m3, m15, m3 |
| movq [dstq+strideq*0], xm0 |
| movhps [dstq+strideq*1], xm0 |
| vextracti32x4 xm2, ym0, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| mova [maskq], xm3 |
| cmp hd, 8 |
| jl .w4_end |
| vextracti32x4 xm2, m0, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm0, m0, 3 |
| movq [dstq+strideq*2], xm0 |
| movhps [dstq+stride3q ], xm0 |
| je .w4_end |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm1 |
| movhps [dstq+strideq*1], xm1 |
| vextracti32x4 xm2, ym1, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| vextracti32x4 xm2, m1, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm1, m1, 3 |
| movq [dstq+strideq*2], xm1 |
| movhps [dstq+stride3q ], xm1 |
| .w4_end: |
| RET |
| .w8: |
| mova m8, [w_mask_shuf8] |
| vpbroadcastd m9, [pb_64] |
| jmp .w8_start |
| .w8_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| add maskq, 16 |
| .w8_start: |
| vpermt2b m2, m8, m3 |
| mova m3, m14 |
| vpdpbusd m3, m2, m9 |
| vpermb m3, m15, m3 |
| mova [dstq+strideq*0], xm0 |
| vextracti32x4 [dstq+strideq*1], ym0, 1 |
| vextracti32x4 [dstq+strideq*2], m0, 2 |
| vextracti32x4 [dstq+stride3q ], m0, 3 |
| mova [maskq], xm3 |
| sub hd, 8 |
| jl .w8_end |
| lea dstq, [dstq+strideq*4] |
| mova [dstq+strideq*0], xm1 |
| vextracti32x4 [dstq+strideq*1], ym1, 1 |
| vextracti32x4 [dstq+strideq*2], m1, 2 |
| vextracti32x4 [dstq+stride3q ], m1, 3 |
| jg .w8_loop |
| .w8_end: |
| RET |
| .w16: |
| mova m8, [w_mask_shuf16] |
| vpbroadcastd m9, [pb_64] |
| jmp .w16_start |
| .w16_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| add maskq, 16 |
| .w16_start: |
| vpermt2b m2, m8, m3 |
| mova m3, m14 |
| vpdpbusd m3, m2, m9 |
| vpermb m3, m15, m3 |
| mova [dstq+strideq*0], ym0 |
| vextracti32x8 [dstq+strideq*1], m0, 1 |
| mova [dstq+strideq*2], ym1 |
| vextracti32x8 [dstq+stride3q ], m1, 1 |
| mova [maskq], xm3 |
| sub hd, 4 |
| jg .w16_loop |
| RET |
| .w32_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| add maskq, 32 |
| .w32: |
| paddw m2, m3 |
| mova m8, m14 |
| vpdpwssd m8, m11, m2 |
| mova [dstq+strideq*0], m0 |
| mova [dstq+strideq*1], m1 |
| call .main |
| paddw m2, m3 |
| mova m3, m14 |
| vpdpwssd m3, m11, m2 |
| vpermt2b m8, m15, m3 |
| mova [dstq+strideq*2], m0 |
| mova [dstq+stride3q ], m1 |
| mova [maskq], ym8 |
| sub hd, 4 |
| jg .w32_loop |
| RET |
| .w64_loop: |
| call .main |
| lea dstq, [dstq+strideq*2] |
| add maskq, 32 |
| .w64: |
| mova m8, m2 |
| mova m9, m3 |
| mova [dstq+strideq*0+64*0], m0 |
| mova [dstq+strideq*0+64*1], m1 |
| call .main |
| paddw m8, m2 |
| paddw m9, m3 |
| mova m2, m14 |
| vpdpwssd m2, m11, m8 |
| mova m3, m14 |
| vpdpwssd m3, m11, m9 |
| vpermt2b m2, m15, m3 |
| mova [dstq+strideq*1+64*0], m0 |
| mova [dstq+strideq*1+64*1], m1 |
| mova [maskq], ym2 |
| sub hd, 2 |
| jg .w64_loop |
| RET |
| .w128_loop: |
| call .main |
| lea dstq, [dstq+strideq*2] |
| add maskq, 64 |
| .w128: |
| mova m16, m2 |
| mova m8, m3 |
| mova [dstq+strideq*0+64*0], m0 |
| mova [dstq+strideq*0+64*1], m1 |
| call .main |
| mova m17, m2 |
| mova m9, m3 |
| mova [dstq+strideq*0+64*2], m0 |
| mova [dstq+strideq*0+64*3], m1 |
| call .main |
| paddw m2, m16 |
| paddw m3, m8 |
| mova m16, m14 |
| vpdpwssd m16, m11, m2 |
| mova m8, m14 |
| vpdpwssd m8, m11, m3 |
| mova [dstq+strideq*1+64*0], m0 |
| mova [dstq+strideq*1+64*1], m1 |
| call .main |
| paddw m2, m17 |
| paddw m3, m9 |
| mova m17, m14 |
| vpdpwssd m17, m11, m2 |
| mova m9, m14 |
| vpdpwssd m9, m11, m3 |
| vpermt2b m16, m15, m8 |
| vpermt2b m17, m15, m9 |
| mova [dstq+strideq*1+64*2], m0 |
| mova [dstq+strideq*1+64*3], m1 |
| mova [maskq+32*0], ym16 |
| mova [maskq+32*1], ym17 |
| sub hd, 2 |
| jg .w128_loop |
| vzeroupper |
| RET |
| ALIGN function_align |
| .main: |
| mova m1, [tmp1q+64*0] |
| mova m3, [tmp2q+64*0] |
| mova m4, [tmp1q+64*1] |
| mova m7, [tmp2q+64*1] |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| psubsw m6, m1, m3 |
| punpcklwd m5, m3, m1 |
| pabsw m6, m6 |
| punpckhwd m3, m1 |
| psubusw m6, m10, m6 |
| psrlw m6, 10 ; 64-m |
| psubw m2, m11, m6 ; m |
| punpcklwd m1, m6, m2 |
| punpckhwd m6, m2 |
| mova m0, m12 |
| vpdpwssd m0, m5, m1 |
| mova m1, m12 |
| vpdpwssd m1, m3, m6 |
| psubsw m5, m4, m7 |
| punpcklwd m6, m7, m4 |
| pabsw m5, m5 |
| punpckhwd m7, m4 |
| psubusw m5, m10, m5 |
| psrlw m5, 10 |
| psubw m3, m11, m5 |
| punpcklwd m4, m5, m3 |
| psrad m0, 4 |
| punpckhwd m5, m3 |
| psrad m1, 4 |
| packusdw m0, m1 |
| mova m1, m12 |
| vpdpwssd m1, m6, m4 |
| mova m4, m12 |
| vpdpwssd m4, m7, m5 |
| psrad m1, 4 |
| psrad m4, 4 |
| packusdw m1, m4 |
| vpsrlvw m0, m13 |
| vpsrlvw m1, m13 |
| ret |
| |
| cglobal w_mask_422_16bpc, 4, 8, 15, dst, stride, tmp1, tmp2, w, h, mask, stride3 |
| %define base r7-w_mask_422_avx512icl_table |
| lea r7, [w_mask_422_avx512icl_table] |
| tzcnt wd, wm |
| mov r6d, r8m ; pixel_max |
| movifnidn hd, hm |
| shr r6d, 11 |
| movsxd wq, [r7+wq*4] |
| vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 |
| vpbroadcastd m9, [base+pw_64] |
| vpbroadcastd m10, [base+mask_round+r6*4] |
| vpbroadcastd m11, [base+bidir_shift+r6*4] |
| mov r6d, r7m ; sign |
| vpbroadcastd m12, [base+w_mask_round+r6*4] |
| mova ym13, [w_mask_end42x] |
| mov maskq, maskmp |
| add wq, r7 |
| paddw m14, m9, m9 ; pw_128 |
| call .main |
| lea stride3q, [strideq*3] |
| jmp wq |
| .w4: |
| movq [dstq+strideq*0], xm0 |
| movhps [dstq+strideq*1], xm0 |
| vextracti32x4 xm2, ym0, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| cmp hd, 8 |
| jl .w4_end |
| vextracti32x4 xm2, m0, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm0, m0, 3 |
| movq [dstq+strideq*2], xm0 |
| movhps [dstq+stride3q ], xm0 |
| je .w4_end |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm1 |
| movhps [dstq+strideq*1], xm1 |
| vextracti32x4 xm2, ym1, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| vextracti32x4 xm2, m1, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm1, m1, 3 |
| movq [dstq+strideq*2], xm1 |
| movhps [dstq+stride3q ], xm1 |
| .w4_end: |
| RET |
| .w8_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w8: |
| mova [dstq+strideq*0], xm0 |
| vextracti32x4 [dstq+strideq*1], ym0, 1 |
| vextracti32x4 [dstq+strideq*2], m0, 2 |
| vextracti32x4 [dstq+stride3q ], m0, 3 |
| sub hd, 8 |
| jl .w8_end |
| lea dstq, [dstq+strideq*4] |
| mova [dstq+strideq*0], xm1 |
| vextracti32x4 [dstq+strideq*1], ym1, 1 |
| vextracti32x4 [dstq+strideq*2], m1, 2 |
| vextracti32x4 [dstq+stride3q ], m1, 3 |
| jg .w8_loop |
| .w8_end: |
| RET |
| .w16_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w16: |
| mova [dstq+strideq*0], ym0 |
| vextracti32x8 [dstq+strideq*1], m0, 1 |
| mova [dstq+strideq*2], ym1 |
| vextracti32x8 [dstq+stride3q ], m1, 1 |
| sub hd, 4 |
| jg .w16_loop |
| RET |
| .w32_loop: |
| call .main |
| lea dstq, [dstq+strideq*2] |
| .w32: |
| mova [dstq+strideq*0], m0 |
| mova [dstq+strideq*1], m1 |
| sub hd, 2 |
| jg .w32_loop |
| RET |
| .w64_loop: |
| call .main |
| add dstq, strideq |
| .w64: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| dec hd |
| jg .w64_loop |
| RET |
| .w128_loop: |
| call .main |
| add dstq, strideq |
| .w128: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| call .main |
| mova [dstq+64*2], m0 |
| mova [dstq+64*3], m1 |
| dec hd |
| jg .w128_loop |
| RET |
| ALIGN function_align |
| .main: |
| mova m1, [tmp1q+64*0] |
| mova m3, [tmp2q+64*0] |
| mova m4, [tmp1q+64*1] |
| mova m7, [tmp2q+64*1] |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| psubsw m6, m1, m3 |
| punpcklwd m5, m3, m1 |
| pabsw m6, m6 |
| punpckhwd m3, m1 |
| psubusw m6, m8, m6 |
| psrlw m6, 10 |
| psubw m2, m9, m6 |
| punpcklwd m1, m6, m2 |
| punpckhwd m6, m2 |
| mova m0, m10 |
| vpdpwssd m0, m5, m1 |
| mova m1, m10 |
| vpdpwssd m1, m3, m6 |
| psubsw m5, m4, m7 |
| punpcklwd m6, m7, m4 |
| pabsw m5, m5 |
| punpckhwd m7, m4 |
| psubusw m5, m8, m5 |
| psrlw m5, 10 |
| psubw m3, m9, m5 |
| punpcklwd m4, m5, m3 |
| psrad m0, 4 |
| punpckhwd m5, m3 |
| psrad m1, 4 |
| packusdw m0, m1 |
| mova m1, m10 |
| vpdpwssd m1, m6, m4 |
| mova m4, m10 |
| vpdpwssd m4, m7, m5 |
| mova m5, m12 |
| vpdpwssd m5, m14, m2 |
| mova m2, m12 |
| vpdpwssd m2, m14, m3 |
| psrad m1, 4 |
| psrad m4, 4 |
| packusdw m1, m4 |
| vpermt2b m5, m13, m2 |
| vpsrlvw m0, m11 |
| vpsrlvw m1, m11 |
| mova [maskq], ym5 |
| add maskq, 32 |
| ret |
| |
| cglobal w_mask_444_16bpc, 4, 8, 13, dst, stride, tmp1, tmp2, w, h, mask, stride3 |
| %define base r7-w_mask_444_avx512icl_table |
| lea r7, [w_mask_444_avx512icl_table] |
| tzcnt wd, wm |
| mov r6d, r8m ; pixel_max |
| movifnidn hd, hm |
| shr r6d, 11 |
| movsxd wq, [r7+wq*4] |
| vpbroadcastd m8, [base+pw_27615] ; ((64 - 38) << 10) + 1023 - 32 |
| vpbroadcastd m9, [base+pw_64] |
| vpbroadcastd m10, [base+mask_round+r6*4] |
| mova m11, [w_mask_end444] |
| vpbroadcastd m12, [base+bidir_shift+r6*4] |
| mov maskq, maskmp |
| add wq, r7 |
| call .main |
| lea stride3q, [strideq*3] |
| jmp wq |
| .w4: |
| movq [dstq+strideq*0], xm0 |
| movhps [dstq+strideq*1], xm0 |
| vextracti32x4 xm2, ym0, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| cmp hd, 8 |
| jl .w4_end |
| vextracti32x4 xm2, m0, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm0, m0, 3 |
| movq [dstq+strideq*2], xm0 |
| movhps [dstq+stride3q ], xm0 |
| je .w4_end |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm1 |
| movhps [dstq+strideq*1], xm1 |
| vextracti32x4 xm2, ym1, 1 |
| movq [dstq+strideq*2], xm2 |
| movhps [dstq+stride3q ], xm2 |
| vextracti32x4 xm2, m1, 2 |
| lea dstq, [dstq+strideq*4] |
| movq [dstq+strideq*0], xm2 |
| movhps [dstq+strideq*1], xm2 |
| vextracti32x4 xm1, m1, 3 |
| movq [dstq+strideq*2], xm1 |
| movhps [dstq+stride3q ], xm1 |
| .w4_end: |
| RET |
| .w8_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w8: |
| mova [dstq+strideq*0], xm0 |
| vextracti32x4 [dstq+strideq*1], ym0, 1 |
| vextracti32x4 [dstq+strideq*2], m0, 2 |
| vextracti32x4 [dstq+stride3q ], m0, 3 |
| sub hd, 8 |
| jl .w8_end |
| lea dstq, [dstq+strideq*4] |
| mova [dstq+strideq*0], xm1 |
| vextracti32x4 [dstq+strideq*1], ym1, 1 |
| vextracti32x4 [dstq+strideq*2], m1, 2 |
| vextracti32x4 [dstq+stride3q ], m1, 3 |
| jg .w8_loop |
| .w8_end: |
| RET |
| .w16_loop: |
| call .main |
| lea dstq, [dstq+strideq*4] |
| .w16: |
| mova [dstq+strideq*0], ym0 |
| vextracti32x8 [dstq+strideq*1], m0, 1 |
| mova [dstq+strideq*2], ym1 |
| vextracti32x8 [dstq+stride3q ], m1, 1 |
| sub hd, 4 |
| jg .w16_loop |
| RET |
| .w32_loop: |
| call .main |
| lea dstq, [dstq+strideq*2] |
| .w32: |
| mova [dstq+strideq*0], m0 |
| mova [dstq+strideq*1], m1 |
| sub hd, 2 |
| jg .w32_loop |
| RET |
| .w64_loop: |
| call .main |
| add dstq, strideq |
| .w64: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| dec hd |
| jg .w64_loop |
| RET |
| .w128_loop: |
| call .main |
| add dstq, strideq |
| .w128: |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| call .main |
| mova [dstq+64*2], m0 |
| mova [dstq+64*3], m1 |
| dec hd |
| jg .w128_loop |
| RET |
| ALIGN function_align |
| .main: |
| mova m1, [tmp1q+64*0] |
| mova m3, [tmp2q+64*0] |
| mova m4, [tmp1q+64*1] |
| mova m7, [tmp2q+64*1] |
| add tmp1q, 64*2 |
| add tmp2q, 64*2 |
| psubsw m6, m1, m3 |
| punpcklwd m5, m3, m1 |
| pabsw m6, m6 |
| punpckhwd m3, m1 |
| psubusw m6, m8, m6 |
| psrlw m6, 10 |
| psubw m2, m9, m6 |
| punpcklwd m1, m6, m2 |
| punpckhwd m6, m2 |
| mova m0, m10 |
| vpdpwssd m0, m5, m1 |
| mova m1, m10 |
| vpdpwssd m1, m3, m6 |
| psubsw m5, m4, m7 |
| punpcklwd m6, m7, m4 |
| pabsw m5, m5 |
| punpckhwd m7, m4 |
| psubusw m5, m8, m5 |
| psrlw m5, 10 |
| psubw m3, m9, m5 |
| punpcklwd m4, m5, m3 |
| psrad m0, 4 |
| punpckhwd m5, m3 |
| psrad m1, 4 |
| packusdw m0, m1 |
| mova m1, m10 |
| vpdpwssd m1, m6, m4 |
| mova m4, m10 |
| vpdpwssd m4, m7, m5 |
| vpermt2b m2, m11, m3 |
| psrad m1, 4 |
| psrad m4, 4 |
| packusdw m1, m4 |
| vpsrlvw m0, m12 |
| vpsrlvw m1, m12 |
| mova [maskq], m2 |
| add maskq, 64 |
| ret |
| |
| cglobal blend_16bpc, 3, 7, 7, dst, ds, tmp, w, h, mask |
| %define base r6-blend_avx512icl_table |
| lea r6, [blend_avx512icl_table] |
| tzcnt wd, wm |
| movifnidn hd, hm |
| movsxd wq, [r6+wq*4] |
| movifnidn maskq, maskmp |
| vpbroadcastd m6, [base+pw_m512] |
| add wq, r6 |
| lea r6, [dsq*3] |
| jmp wq |
| .w4: |
| pmovzxbw ym19, [maskq] |
| movq xm16, [dstq+dsq*0] |
| movhps xm16, [dstq+dsq*1] |
| vpbroadcastq ym17, [dstq+dsq*2] |
| vpbroadcastq ym18, [dstq+r6 ] |
| pmullw ym19, ym6 |
| vpblendd ym16, ym17, 0x30 |
| vpblendd ym16, ym18, 0xc0 |
| psubw ym17, ym16, [tmpq] |
| add maskq, 16 |
| add tmpq, 32 |
| pmulhrsw ym17, ym19 |
| paddw ym16, ym17 |
| vextracti128 xm17, ym16, 1 |
| movq [dstq+dsq*0], xm16 |
| movhps [dstq+dsq*1], xm16 |
| movq [dstq+dsq*2], xm17 |
| movhps [dstq+r6 ], xm17 |
| lea dstq, [dstq+dsq*4] |
| sub hd, 4 |
| jg .w4 |
| vzeroupper |
| RET |
| .w8: |
| pmovzxbw m2, [maskq] |
| mova xm0, [dstq+dsq*0] |
| vinserti32x4 ym0, [dstq+dsq*1], 1 |
| vinserti32x4 m0, [dstq+dsq*2], 2 |
| vinserti32x4 m0, [dstq+r6 ], 3 |
| pmullw m2, m6 |
| psubw m1, m0, [tmpq] |
| add maskq, 32 |
| add tmpq, 64 |
| pmulhrsw m1, m2 |
| paddw m0, m1 |
| mova [dstq+dsq*0], xm0 |
| vextracti32x4 [dstq+dsq*1], ym0, 1 |
| vextracti32x4 [dstq+dsq*2], m0, 2 |
| vextracti32x4 [dstq+r6 ], m0, 3 |
| lea dstq, [dstq+dsq*4] |
| sub hd, 4 |
| jg .w8 |
| RET |
| .w16: |
| pmovzxbw m4, [maskq+32*0] |
| pmovzxbw m5, [maskq+32*1] |
| mova ym0, [dstq+dsq*0] |
| vinserti32x8 m0, [dstq+dsq*1], 1 |
| mova ym1, [dstq+dsq*2] |
| vinserti32x8 m1, [dstq+r6 ], 1 |
| pmullw m4, m6 |
| pmullw m5, m6 |
| psubw m2, m0, [tmpq+64*0] |
| psubw m3, m1, [tmpq+64*1] |
| add maskq, 32*2 |
| add tmpq, 64*2 |
| pmulhrsw m2, m4 |
| pmulhrsw m3, m5 |
| paddw m0, m2 |
| paddw m1, m3 |
| mova [dstq+dsq*0], ym0 |
| vextracti32x8 [dstq+dsq*1], m0, 1 |
| mova [dstq+dsq*2], ym1 |
| vextracti32x8 [dstq+r6 ], m1, 1 |
| lea dstq, [dstq+dsq*4] |
| sub hd, 4 |
| jg .w16 |
| RET |
| .w32: |
| pmovzxbw m4, [maskq+32*0] |
| pmovzxbw m5, [maskq+32*1] |
| mova m0, [dstq+dsq*0] |
| mova m1, [dstq+dsq*1] |
| pmullw m4, m6 |
| pmullw m5, m6 |
| psubw m2, m0, [tmpq+ 64*0] |
| psubw m3, m1, [tmpq+ 64*1] |
| add maskq, 32*2 |
| add tmpq, 64*2 |
| pmulhrsw m2, m4 |
| pmulhrsw m3, m5 |
| paddw m0, m2 |
| paddw m1, m3 |
| mova [dstq+dsq*0], m0 |
| mova [dstq+dsq*1], m1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w32 |
| RET |
| |
| cglobal blend_v_16bpc, 3, 6, 5, dst, ds, tmp, w, h |
| lea r5, [blend_v_avx512icl_table] |
| tzcnt wd, wm |
| movifnidn hd, hm |
| movsxd wq, [r5+wq*4] |
| add wq, r5 |
| jmp wq |
| .w2: |
| vpbroadcastd xmm2, [obmc_masks_avx2+2*2] |
| .w2_loop: |
| movd xmm0, [dstq+dsq*0] |
| pinsrd xmm0, [dstq+dsq*1], 1 |
| movq xmm1, [tmpq] |
| add tmpq, 4*2 |
| psubw xmm1, xmm0, xmm1 |
| pmulhrsw xmm1, xmm2 |
| paddw xmm0, xmm1 |
| movd [dstq+dsq*0], xmm0 |
| pextrd [dstq+dsq*1], xmm0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w2_loop |
| RET |
| .w4: |
| vpbroadcastq xmm2, [obmc_masks_avx2+4*2] |
| .w4_loop: |
| movq xmm0, [dstq+dsq*0] |
| movhps xmm0, [dstq+dsq*1] |
| psubw xmm1, xmm0, [tmpq] |
| add tmpq, 8*2 |
| pmulhrsw xmm1, xmm2 |
| paddw xmm0, xmm1 |
| movq [dstq+dsq*0], xmm0 |
| movhps [dstq+dsq*1], xmm0 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w4_loop |
| RET |
| .w8: |
| vbroadcasti32x4 ym2, [obmc_masks_avx2+8*2] |
| .w8_loop: |
| mova xm0, [dstq+dsq*0] |
| vinserti32x4 ym0, [dstq+dsq*1], 1 |
| psubw ym1, ym0, [tmpq] |
| add tmpq, 16*2 |
| pmulhrsw ym1, ym2 |
| paddw ym0, ym1 |
| mova [dstq+dsq*0], xm0 |
| vextracti32x4 [dstq+dsq*1], ym0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w8_loop |
| RET |
| .w16: |
| vbroadcasti32x8 m2, [obmc_masks_avx2+16*2] |
| .w16_loop: |
| mova ym0, [dstq+dsq*0] |
| vinserti32x8 m0, [dstq+dsq*1], 1 |
| psubw m1, m0, [tmpq] |
| add tmpq, 32*2 |
| pmulhrsw m1, m2 |
| paddw m0, m1 |
| mova [dstq+dsq*0], ym0 |
| vextracti32x8 [dstq+dsq*1], m0, 1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w16_loop |
| RET |
| .w32: |
| mova m4, [obmc_masks_avx2+32*2] |
| .w32_loop: |
| mova m0, [dstq+dsq*0] |
| psubw m2, m0, [tmpq+ 64*0] |
| mova m1, [dstq+dsq*1] |
| psubw m3, m1, [tmpq+ 64*1] |
| add tmpq, 64*2 |
| pmulhrsw m2, m4 |
| pmulhrsw m3, m4 |
| paddw m0, m2 |
| paddw m1, m3 |
| mova [dstq+dsq*0], m0 |
| mova [dstq+dsq*1], m1 |
| lea dstq, [dstq+dsq*2] |
| sub hd, 2 |
| jg .w32_loop |
| RET |
| |
| cglobal blend_h_16bpc, 3, 7, 9, dst, ds, tmp, w, h, mask |
| %define base r6-$$ |
| lea r6, [$$] |
| tzcnt wd, wm |
| mov hd, hm |
| movsxd wq, [base+blend_h_avx512icl_table+wq*4] |
| lea maskq, [base+obmc_masks_avx2+hq*2] |
| lea hd, [hq*3] |
| lea wq, [base+blend_h_avx512icl_table+wq] |
| shr hd, 2 ; h * 3/4 |
| lea maskq, [maskq+hq*2] |
| neg hq |
| jmp wq |
| .w2: |
| movd xmm0, [dstq+dsq*0] |
| pinsrd xmm0, [dstq+dsq*1], 1 |
| movd xmm2, [maskq+hq*2] |
| movq xmm1, [tmpq] |
| add tmpq, 4*2 |
| punpcklwd xmm2, xmm2 |
| psubw xmm1, xmm0, xmm1 |
| pmulhrsw xmm1, xmm2 |
| paddw xmm0, xmm1 |
| movd [dstq+dsq*0], xmm0 |
| pextrd [dstq+dsq*1], xmm0, 1 |
| lea dstq, [dstq+dsq*2] |
| add hq, 2 |
| jl .w2 |
| RET |
| .w4: |
| mova xmm3, [blend_shuf] |
| .w4_loop: |
| movq xmm0, [dstq+dsq*0] |
| movhps xmm0, [dstq+dsq*1] |
| movd xmm2, [maskq+hq*2] |
| psubw xmm1, xmm0, [tmpq] |
| add tmpq, 8*2 |
| pshufb xmm2, xmm3 |
| pmulhrsw xmm1, xmm2 |
| paddw xmm0, xmm1 |
| movq [dstq+dsq*0], xmm0 |
| movhps [dstq+dsq*1], xmm0 |
| lea dstq, [dstq+dsq*2] |
| add hq, 2 |
| jl .w4_loop |
| RET |
| .w8: |
| vbroadcasti32x4 ym3, [blend_shuf] |
| shufpd ym3, ym3, 0x0c |
| .w8_loop: |
| mova xm0, [dstq+dsq*0] |
| vinserti32x4 ym0, [dstq+dsq*1], 1 |
| vpbroadcastd ym2, [maskq+hq*2] |
| psubw ym1, ym0, [tmpq] |
| add tmpq, 16*2 |
| pshufb ym2, ym3 |
| pmulhrsw ym1, ym2 |
| paddw ym0, ym1 |
| mova [dstq+dsq*0], xm0 |
| vextracti32x4 [dstq+dsq*1], ym0, 1 |
| lea dstq, [dstq+dsq*2] |
| add hq, 2 |
| jl .w8_loop |
| RET |
| .w16: |
| vbroadcasti32x4 m3, [blend_shuf] |
| shufpd m3, m3, 0xf0 |
| .w16_loop: |
| mova ym0, [dstq+dsq*0] |
| vinserti32x8 m0, [dstq+dsq*1], 1 |
| vpbroadcastd m2, [maskq+hq*2] |
| psubw m1, m0, [tmpq] |
| add tmpq, 32*2 |
| pshufb m2, m3 |
| pmulhrsw m1, m2 |
| paddw m0, m1 |
| mova [dstq+dsq*0], ym0 |
| vextracti32x8 [dstq+dsq*1], m0, 1 |
| lea dstq, [dstq+dsq*2] |
| add hq, 2 |
| jl .w16_loop |
| RET |
| .w32: |
| vpbroadcastw m4, [maskq+hq*2] |
| vpbroadcastw m5, [maskq+hq*2+2] |
| mova m0, [dstq+dsq*0] |
| psubw m2, m0, [tmpq+ 64*0] |
| mova m1, [dstq+dsq*1] |
| psubw m3, m1, [tmpq+ 64*1] |
| add tmpq, 64*2 |
| pmulhrsw m2, m4 |
| pmulhrsw m3, m5 |
| paddw m0, m2 |
| paddw m1, m3 |
| mova [dstq+dsq*0], m0 |
| mova [dstq+dsq*1], m1 |
| lea dstq, [dstq+dsq*2] |
| add hq, 2 |
| jl .w32 |
| RET |
| .w64: |
| vpbroadcastw m4, [maskq+hq*2] |
| mova m0, [dstq+64*0] |
| psubw m2, m0, [tmpq+64*0] |
| mova m1, [dstq+64*1] |
| psubw m3, m1, [tmpq+64*1] |
| add tmpq, 64*2 |
| pmulhrsw m2, m4 |
| pmulhrsw m3, m4 |
| paddw m0, m2 |
| paddw m1, m3 |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| add dstq, dsq |
| inc hq |
| jl .w64 |
| RET |
| .w128: |
| vpbroadcastw m8, [maskq+hq*2] |
| mova m0, [dstq+64*0] |
| psubw m4, m0, [tmpq+64*0] |
| mova m1, [dstq+64*1] |
| psubw m5, m1, [tmpq+64*1] |
| mova m2, [dstq+64*2] |
| psubw m6, m2, [tmpq+64*2] |
| mova m3, [dstq+64*3] |
| psubw m7, m3, [tmpq+64*3] |
| add tmpq, 64*4 |
| REPX {pmulhrsw x, m8}, m4, m5, m6, m7 |
| paddw m0, m4 |
| paddw m1, m5 |
| paddw m2, m6 |
| paddw m3, m7 |
| mova [dstq+64*0], m0 |
| mova [dstq+64*1], m1 |
| mova [dstq+64*2], m2 |
| mova [dstq+64*3], m3 |
| add dstq, dsq |
| inc hq |
| jl .w128 |
| RET |
| |
| cglobal resize_16bpc, 6, 12, 32, dst, dst_stride, src, src_stride, \ |
| dst_w, h, src_w, dx, mx0, pxmax |
| sub dword mx0m, 4<<14 |
| sub dword src_wm, 8 |
| mov r6, ~0 |
| vpbroadcastd m5, dxm |
| vpbroadcastd m8, mx0m |
| vpbroadcastd m6, src_wm |
| kmovq k6, r6 |
| DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, _, _, pxmax |
| LEA r7, $$ |
| %define base r7-$$ |
| vpbroadcastd m3, [base+pd_16384] |
| vpbroadcastd m7, [base+pd_63] |
| mova m24, [base+resize_permA] |
| mova m25, [base+resize_permB] |
| mova m26, [base+resize_permC] |
| mova m27, [base+resize_permD] |
| vbroadcasti32x4 m28, [base+resize_shufA] |
| vbroadcasti32x4 m29, [base+resize_shufB] |
| mova m30, [base+resize_permE] |
| vpbroadcastw ym31, pxmaxm |
| vpdpwssd m8, m5, [base+rescale_mul] ; mx+dx*[0-15] |
| pslld m5, 4 ; dx*16 |
| pslld m6, 14 |
| pxor m2, m2 |
| .loop_y: |
| xor xd, xd |
| mova m4, m8 ; per-line working version of mx |
| .loop_x: |
| pmaxsd m0, m4, m2 |
| psrad m9, m4, 8 ; filter offset (unmasked) |
| pminsd m0, m6 ; iclip(mx, 0, src_w-8) |
| psubd m1, m4, m0 ; pshufb offset |
| psrad m0, 14 ; clipped src_x offset |
| psrad m1, 14 ; pshufb edge_emu offset |
| vptestmd k5, m1, m1 |
| pand m9, m7 ; filter offset (masked) |
| ktestw k5, k5 |
| jz .load |
| vpbroadcastq m14, [base+pd_0_4] |
| vpermq m10, m0, q1100 |
| vpermq m11, m0, q3322 |
| vpermq m20, m1, q1100 |
| vpermq m21, m1, q3322 |
| punpckldq m10, m10 |
| punpckldq m11, m11 |
| punpckldq m20, m20 |
| punpckldq m21, m21 |
| paddd m10, m14 |
| paddd m11, m14 |
| paddd m20, m14 |
| paddd m21, m14 |
| vextracti32x8 ym12, m10, 1 |
| vextracti32x8 ym13, m11, 1 |
| vextracti32x8 ym22, m20, 1 |
| vextracti32x8 ym23, m21, 1 |
| kmovq k1, k6 |
| kmovq k2, k6 |
| kmovq k3, k6 |
| kmovq k4, k6 |
| vpgatherdq m16{k1}, [srcq+ym10*2] ; 0 1 2 3 |
| vpgatherdq m17{k2}, [srcq+ym11*2] ; 4 5 6 7 |
| vpgatherdq m18{k3}, [srcq+ym12*2] ; 8 9 A B |
| vpgatherdq m19{k4}, [srcq+ym13*2] ; C D E F |
| kmovq k1, k6 |
| kmovq k2, k6 |
| kmovq k3, k6 |
| kmovq k4, k6 |
| vpgatherdq m0{k1}, [base+resize_shuf+8+ym20*2] |
| vpgatherdq m1{k2}, [base+resize_shuf+8+ym21*2] |
| vpgatherdq m14{k3}, [base+resize_shuf+8+ym22*2] |
| vpgatherdq m15{k4}, [base+resize_shuf+8+ym23*2] |
| pshufb m16, m0 |
| pshufb m17, m1 |
| pshufb m18, m14 |
| pshufb m19, m15 |
| mova m20, m24 |
| mova m22, m24 |
| mova m21, m25 |
| mova m23, m25 |
| vpermi2d m20, m16, m17 ; 0-3a 0-3b 4-7a 4-7b |
| vpermi2d m21, m16, m17 ; 0-3c 0-3d 4-7c 4-7d |
| vpermi2d m22, m18, m19 ; 8-Ba 8-Bb C-Fa C-Fb |
| vpermi2d m23, m18, m19 ; 8-Bc 8-Bd C-Fc C-Fd |
| mova m15, m26 |
| mova m17, m26 |
| mova m16, m27 |
| mova m18, m27 |
| vpermi2q m15, m20, m22 ; 0-3a 4-7a 8-Ba C-Fa |
| vpermi2q m16, m20, m22 ; 0-3b 4-7b 8-Bb C-Fb |
| vpermi2q m17, m21, m23 ; 0-3c 4-7c 8-Bc C-Fc |
| vpermi2q m18, m21, m23 ; 0-3d 4-7d 8-Bd C-Fd |
| kmovq k1, k6 |
| kmovq k2, k6 |
| vpgatherdd m11{k1}, [base+resize_filter+m9*8+0] |
| vpgatherdd m13{k2}, [base+resize_filter+m9*8+4] |
| pshufb m10, m11, m28 |
| pshufb m11, m11, m29 |
| pshufb m12, m13, m28 |
| pshufb m13, m13, m29 |
| jmp .filter |
| .load: |
| kmovq k1, k6 |
| kmovq k2, k6 |
| kmovq k3, k6 |
| kmovq k4, k6 |
| vpgatherdd m11{k1}, [base+resize_filter+m9*8+0] |
| vpgatherdd m13{k2}, [base+resize_filter+m9*8+4] |
| pshufb m10, m11, m28 |
| pshufb m11, m11, m29 |
| pshufb m12, m13, m28 |
| pshufb m13, m13, m29 |
| vpgatherdd m15{k3}, [srcq+m0*2+ 0] |
| vpgatherdd m16{k4}, [srcq+m0*2+ 4] |
| kmovq k1, k6 |
| kmovq k2, k6 |
| vpgatherdd m17{k1}, [srcq+m0*2+ 8] |
| vpgatherdd m18{k2}, [srcq+m0*2+12] |
| .filter: |
| mova m14, m2 |
| vpdpwssd m14, m15, m10 |
| vpdpwssd m14, m16, m11 |
| vpdpwssd m14, m17, m12 |
| vpdpwssd m14, m18, m13 |
| psubd m14, m3, m14 |
| psrad m14, 15 |
| packusdw m14, m14 |
| vpermq m14, m30, m14 |
| pminsw ym14, ym31 |
| mova [dstq+xq*2], ym14 |
| paddd m4, m5 |
| add xd, 16 |
| cmp xd, dst_wd |
| jl .loop_x |
| add dstq, dst_strideq |
| add srcq, src_strideq |
| dec hd |
| jg .loop_y |
| RET |
| |
| %endif ; ARCH_X86_64 |
