; Copyright © 2020, VideoLAN and dav1d authors
; Copyright © 2020, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

%if ARCH_X86_64

SECTION_RODATA 64

obmc_masks:
pw_512:         times 2 dw 512
                ; 2
                db 45, 19, 64,  0
                ; 4
                db 39, 25, 50, 14, 59,  5, 64,  0
                ; 8
                db 36, 28, 42, 22, 48, 16, 53, 11, 57,  7, 61,  3, 64,  0, 64,  0
                ; 16
                db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10
                db 56,  8, 58,  6, 60,  4, 61,  3, 64,  0, 64,  0, 64,  0, 64,  0
                ; 32
                db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20
                db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55,  9
                db 56,  8, 57,  7, 58,  6, 59,  5, 60,  4, 60,  4, 61,  3, 62,  2
                db 64,  0, 64,  0, 64,  0, 64,  0, 64,  0, 64,  0, 64,  0, 64,  0

warp_8x8_permA: db  4,  5,  6,  7, 16, 17, 18, 19,  5,  6,  7,  8, 17, 18, 19, 20
                db  6,  7,  8,  9, 18, 19, 20, 21,  7,  8,  9, 10, 19, 20, 21, 22
                db  8,  9, 10, 11, 20, 21, 22, 23,  9, 10, 11, 12, 21, 22, 23, 24
                db 10, 11, 12, 13, 22, 23, 24, 25, 11, 12, 13, 14, 23, 24, 25, 26
warp_8x8_permB: db  0,  1,  2,  3, 20, 21, 22, 23,  1,  2,  3,  4, 21, 22, 23, 24
                db  2,  3,  4,  5, 22, 23, 24, 25,  3,  4,  5,  6, 23, 24, 25, 26
                db  4,  5,  6,  7, 24, 25, 26, 27,  5,  6,  7,  8, 25, 26, 27, 28
                db  6,  7,  8,  9, 26, 27, 28, 29,  7,  8,  9, 10, 27, 28, 29, 30
warp_8x8_permC: db -1,  0, -1,  1, -1,  8, -1,  9, -1,  4, -1,  5, -1, 12, -1, 13
warp_8x8_permD: db -1,  2, -1,  3, -1, 10, -1, 11, -1,  6, -1,  7, -1, 14, -1, 15
pd_0to7:        dd  0,  1,  2,  3,  4,  5,  6,  7
warp_8x8_hpack: db  3, 11,  3, 11, 35, 43, 35, 43
pd_16384:       dd 16384
pd_262144:      dd 262144
warp_8x8_end:   db  0,  4, 16, 20, 32, 36, 48, 52,  2,  6, 18, 22, 34, 38, 50, 54
warp_8x8t_end:  db  2,  3, 10, 11, 18, 19, 26, 27, 34, 35, 42, 43, 50, 51, 58, 59
                db  6,  7, 14, 15, 22, 23, 30, 31, 38, 39, 46, 47, 54, 55, 62, 63
bidir_sctr_w4:  dd  0,  1,  8,  9,  2,  3, 10, 11,  4,  5, 12, 13,  6,  7, 14, 15
wm_420_perm4:   db  1,  3,  9, 11,  5,  7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31
                db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63
                db  0,  2,  8, 10,  4,  6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30
                db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62
wm_420_perm8:   db  1,  3, 17, 19,  5,  7, 21, 23,  9, 11, 25, 27, 13, 15, 29, 31
                db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63
                db  0,  2, 16, 18,  4,  6, 20, 22,  8, 10, 24, 26, 12, 14, 28, 30
                db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62
wm_420_perm16:  db  1,  3, 33, 35,  5,  7, 37, 39,  9, 11, 41, 43, 13, 15, 45, 47
                db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63
                db  0,  2, 32, 34,  4,  6, 36, 38,  8, 10, 40, 42, 12, 14, 44, 46
                db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62
wm_420_mask:    db  3,  7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63
                db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127
                db  1,  5,  9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
                db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
wm_422_mask:    db  2,  6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62
                db  1,  5,  9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61
                db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126
                db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125
wm_444_mask:    db  1,  3,  5,  7,  9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
                db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63
                db  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
                db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62
bilin_h_perm16: db  1,  0,  2,  1,  3,  2,  4,  3,  5,  4,  6,  5,  7,  6,  8,  7
                db  9,  8, 10,  9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
                db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39
                db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47
bilin_h_perm32: db  1,  0,  2,  1,  3,  2,  4,  3,  5,  4,  6,  5,  7,  6,  8,  7
                db  9,  8, 10,  9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15
                db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23
                db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31
bilin_v_perm8:  db 16,  0, 17,  1, 18,  2, 19,  3, 20,  4, 21,  5, 22,  6, 23,  7
                db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
                db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87
                db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39
bilin_v_perm16: db 16,  0, 17,  1, 18,  2, 19,  3, 20,  4, 21,  5, 22,  6, 23,  7
                db 24,  8, 25,  9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15
                db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23
                db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31
bilin_v_perm32: db 64,  0, 65,  1, 66,  2, 67,  3, 68,  4, 69,  5, 70,  6, 71,  7
                db 72,  8, 73,  9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15
                db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23
                db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31
bilin_v_perm64: dq  0,  4,  1,  5,  2,  6,  3,  7
spel_h_perm16a: db  0,  1,  2,  3,  1,  2,  3,  4,  2,  3,  4,  5,  3,  4,  5,  6
                db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
                db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
                db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
spel_h_perm16b: db  4,  5,  6,  7,  5,  6,  7,  8,  6,  7,  8,  9,  7,  8,  9, 10
                db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
                db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42
                db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50
spel_h_perm16c: db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
                db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
                db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46
                db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54
spel_h_perm32a: db  0,  1,  2,  3,  1,  2,  3,  4,  2,  3,  4,  5,  3,  4,  5,  6
                db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
                db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
                db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
spel_h_perm32b: db  4,  5,  6,  7,  5,  6,  7,  8,  6,  7,  8,  9,  7,  8,  9, 10
                db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18
                db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26
                db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34
spel_h_perm32c: db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
                db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22
                db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30
                db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38
spel_v_perm16:  db 32,  0, 33,  1, 34,  2, 35,  3, 36,  4, 37,  5, 38,  6, 39,  7
                db  0,  8,  1,  9,  2, 10,  3, 11,  4, 12,  5, 13,  6, 14,  7, 15
                db 40, 16, 41, 17, 42, 18, 43, 19, 44, 20, 45, 21, 46, 22, 47, 23
                db 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31
spel_v_perm32:  db  0, 32,  1, 33,  2, 34,  3, 35,  4, 36,  5, 37,  6, 38,  7, 39
                db  8, 40,  9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
                db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
                db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
spel_hv_perm4a: db  8,  9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23
                db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31
spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39
                db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47
spel_hv_perm4c: db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55
                db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63
spel_hv_perm4d: db 18, 19,  0,  1, 22, 23,  4,  5, 26, 27,  8,  9, 30, 31, 12, 13
                db  0,  1, 16, 17,  4,  5, 20, 21,  8,  9, 24, 25, 12, 13, 28, 29
spel_hv_perm8a: db  0,  1, 16, 17,  2,  3, 18, 19,  4,  5, 20, 21,  6,  7, 22, 23
                db  8,  9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31
                db 16, 17, 32, 33, 18, 19, 34, 35, 20, 21, 36, 37, 22, 23, 38, 39
                db 24, 25, 40, 41, 26, 27, 42, 43, 28, 29, 44, 45, 30, 31, 46, 47
spel_hv_perm8b: db 32, 33, 48, 49, 34, 35, 50, 51, 36, 37, 52, 53, 38, 39, 54, 55
                db 40, 41, 56, 57, 42, 43, 58, 59, 44, 45, 60, 61, 46, 47, 62, 63
                db 48, 49, 64, 65, 50, 51, 66, 67, 52, 53, 68, 69, 54, 55, 70, 71
                db 56, 57, 72, 73, 58, 59, 74, 75, 60, 61, 76, 77, 62, 63, 78, 79
spel_hv_perm8c: db 34, 35,  0,  1, 38, 39,  4,  5, 42, 43,  8,  9, 46, 47, 12, 13
                db 50, 51, 16, 17, 54, 55, 20, 21, 58, 59, 24, 25, 62, 63, 28, 29
                db  0,  1, 32, 33,  4,  5, 36, 37,  8,  9, 40, 41, 12, 13, 44, 45
                db 16, 17, 48, 49, 20, 21, 52, 53, 24, 25, 56, 57, 28, 29, 60, 61
spel_hv_end16:  db  1,  3, 17, 19,  5,  7, 21, 23, 33, 35, 49, 51, 37, 39, 53, 55
                db  9, 11, 25, 27, 13, 15, 29, 31, 41, 43, 57, 59, 45, 47, 61, 63
spel_hv_perm16a:db  0,  1,  2,  3, 32, 33, 34, 35,  1,  2,  3,  4, 33, 34, 35, 36
                db  2,  3,  4,  5, 34, 35, 36, 37,  3,  4,  5,  6, 35, 36, 37, 38
spel_hv_perm16c:db  8,  9, 10, 11, 40, 41, 42, 43,  9, 10, 11, 12, 41, 42, 43, 44
                db 10, 11, 12, 13, 42, 43, 44, 45, 11, 12, 13, 14, 43, 44, 45, 46
                db 16, 17, 18, 19, 48, 49, 50, 51, 17, 18, 19, 20, 49, 50, 51, 52
                db 18, 19, 20, 21, 50, 51, 52, 53, 19, 20, 21, 22, 51, 52, 53, 54
spel_hv_perm16b:db  4,  5,  6,  7, 36, 37, 38, 39,  5,  6,  7,  8, 37, 38, 39, 40
                db  6,  7,  8,  9, 38, 39, 40, 41,  7,  8,  9, 10, 39, 40, 41, 42
                db 12, 13, 14, 15, 44, 45, 46, 47, 13, 14, 15, 16, 45, 46, 47, 48
                db 14, 15, 16, 17, 46, 47, 48, 49, 15, 16, 17, 18, 47, 48, 49, 50
spel_hv_perm16d:db  0,  1,  2,  3,  1,  2,  3,  4,  4,  5,  6,  7,  5,  6,  7,  8
                db  2,  3,  4,  5,  3,  4,  5,  6,  6,  7,  8,  9,  7,  8,  9, 10
                db  8,  9, 10, 11,  9, 10, 11, 12, 12, 13, 14, 15, 13, 14, 15, 16
                db 10, 11, 12, 13, 11, 12, 13, 14, 14, 15, 16, 17, 15, 16, 17, 18
spel_hv_perm16e:db  4,  5,  6,  7,  5,  6,  7,  8,  8,  9, 10, 11,  9, 10, 11, 12
                db  6,  7,  8,  9,  7,  8,  9, 10, 10, 11, 12, 13, 11, 12, 13, 14
                db 12, 13, 14, 15, 13, 14, 15, 16, 16, 17, 18, 19, 17, 18, 19, 20
                db 14, 15, 16, 17, 15, 16, 17, 18, 18, 19, 20, 21, 19, 20, 21, 22
spel_hv_end:    db  1,  3,  5,  7, 17, 19, 21, 23, 33, 35, 37, 39, 49, 51, 53, 55
deint_shuf4:    db  0,  4,  1,  5,  2,  6,  3,  7,  4,  8,  5,  9,  6, 10,  7, 11
subpel_h_shuf4: db  0,  1,  2,  3,  1,  2,  3,  4,  8,  9, 10, 11,  9, 10, 11, 12
                db  2,  3,  4,  5,  3,  4,  5,  6, 10, 11, 12, 13, 11, 12, 13, 14
subpel_h_shufA: db  0,  1,  2,  3,  1,  2,  3,  4,  2,  3,  4,  5,  3,  4,  5,  6
subpel_h_shufB: db  4,  5,  6,  7,  5,  6,  7,  8,  6,  7,  8,  9,  7,  8,  9, 10
subpel_h_shufC: db  8,  9, 10, 11,  9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
bilin_h_shuf4:  db  1,  0,  2,  1,  3,  2,  4,  3,  9,  8, 10,  9, 11, 10, 12, 11
bilin_h_shuf8:  db  1,  0,  2,  1,  3,  2,  4,  3,  5,  4,  6,  5,  7,  6,  8,  7
bilin_v_shuf4:  db  4,  0,  5,  1,  6,  2,  7,  3,  8,  4,  9,  5, 10,  6, 11,  7
blend_shuf:     db  0,  1,  0,  1,  0,  1,  0,  1,  2,  3,  2,  3,  2,  3,  2,  3
rescale_mul:    dd  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15
resize_shuf:    db  0,  0,  0,  0,  0,  1,  2,  3,  4,  5,  6,  7,  7,  7,  7,  7
resize_permA:   dd  0,  2,  4,  6,  8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
resize_permB:   dd  1,  3,  5,  7,  9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
resize_permC:   dd  0,  4,  8, 12
pb_02461357:    db  0,  2,  4,  6,  1,  3,  5,  7

wm_420_perm64:  dq 0xfedcba9876543210
wm_sign:        dd 0x40804080, 0xc0c0c0c0, 0x40404040

pb_8x0_8x8: times 8 db 0
            times 8 db 8
pb_127:     times 4 db 127
pw_m128     times 2 dw -128
pw_m256:    times 2 dw -256
pw_1024:    times 2 dw 1024
pw_2048:    times 2 dw 2048
pw_6903:    times 2 dw 6903
pw_8192:    times 2 dw 8192
pd_32:              dd 32
pd_34:              dd 34
pd_63:              dd 63
pd_512:             dd 512
pd_32768:           dd 32768

%define pb_m64 (wm_sign+4)
%define pb_64  (wm_sign+8)
%define pd_2   (pd_0to7+8)

cextern mc_subpel_filters
%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8)
cextern mc_warp_filter
cextern resize_filter

%macro BASE_JMP_TABLE 3-*
    %xdefine %1_%2_table (%%table - %3)
    %xdefine %%base %1_%2
    %%table:
    %rep %0 - 2
        dw %%base %+ _w%3 - %%base
        %rotate 1
    %endrep
%endmacro

%macro HV_JMP_TABLE 5-*
    %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3)
    %xdefine %%base %1_%3
    %assign %%types %4
    %if %%types & 1
        %xdefine %1_%2_h_%3_table  (%%h  - %5)
        %%h:
        %rep %0 - 4
            dw %%prefix %+ .h_w%5 - %%base
            %rotate 1
        %endrep
        %rotate 4
    %endif
    %if %%types & 2
        %xdefine %1_%2_v_%3_table  (%%v  - %5)
        %%v:
        %rep %0 - 4
            dw %%prefix %+ .v_w%5 - %%base
            %rotate 1
        %endrep
        %rotate 4
    %endif
    %if %%types & 4
        %xdefine %1_%2_hv_%3_table (%%hv - %5)
        %%hv:
        %rep %0 - 4
            dw %%prefix %+ .hv_w%5 - %%base
            %rotate 1
        %endrep
    %endif
%endmacro

%macro BIDIR_JMP_TABLE 2-*
    %xdefine %1_%2_table (%%table - 2*%3)
    %xdefine %%base %1_%2_table
    %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2)
    %%table:
    %rep %0 - 2
        dd %%prefix %+ .w%3 - %%base
        %rotate 1
    %endrep
%endmacro

%xdefine put_avx512icl mangle(private_prefix %+ _put_bilin_8bpc_avx512icl.put)
%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_8bpc_avx512icl.prep)

%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX

BASE_JMP_TABLE put,  avx512icl,         2, 4, 8, 16, 32, 64, 128
BASE_JMP_TABLE prep, avx512icl,            4, 8, 16, 32, 64, 128
HV_JMP_TABLE put,  bilin, avx512icl, 7, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, bilin, avx512icl, 7,    4, 8, 16, 32, 64, 128
HV_JMP_TABLE put,  8tap,  avx512icl, 3, 2, 4, 8, 16, 32, 64, 128
HV_JMP_TABLE prep, 8tap,  avx512icl, 7,    4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE avg, avx512icl,            4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_avg, avx512icl,          4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE mask, avx512icl,           4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_420, avx512icl,     4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_422, avx512icl,     4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE w_mask_444, avx512icl,     4, 8, 16, 32, 64, 128
BIDIR_JMP_TABLE blend, avx512icl,          4, 8, 16, 32
BIDIR_JMP_TABLE blend_v, avx512icl,     2, 4, 8, 16, 32
BIDIR_JMP_TABLE blend_h, avx512icl,     2, 4, 8, 16, 32, 64, 128

SECTION .text

%macro WRAP_YMM 1+
INIT_YMM cpuname
    %1
INIT_ZMM cpuname
%endmacro

INIT_ZMM avx512icl
cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy
    movifnidn          mxyd, r6m ; mx
    lea                  r7, [put_avx512icl]
    tzcnt                wd, wm
    movifnidn            hd, hm
    test               mxyd, mxyd
    jnz .h
    mov                mxyd, r7m ; my
    test               mxyd, mxyd
    jnz .v
.put:
    movzx                wd, word [r7+wq*2+table_offset(put,)]
    add                  wq, r7
    jmp                  wq
.put_w2:
    movzx               r6d, word [srcq+ssq*0]
    movzx               r7d, word [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6w
    mov        [dstq+dsq*1], r7w
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w2
    RET
.put_w4:
    mov                 r6d, [srcq+ssq*0]
    mov                 r7d, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6d
    mov        [dstq+dsq*1], r7d
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w4
    RET
.put_w8:
    mov                  r6, [srcq+ssq*0]
    mov                  r7, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mov        [dstq+dsq*0], r6
    mov        [dstq+dsq*1], r7
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w8
    RET
.put_w16:
    movu               xmm0, [srcq+ssq*0]
    movu               xmm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mova       [dstq+dsq*0], xmm0
    mova       [dstq+dsq*1], xmm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w16
    RET
.put_w32:
    movu                ym0, [srcq+ssq*0]
    movu                ym1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mova       [dstq+dsq*0], ym0
    mova       [dstq+dsq*1], ym1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w32
    RET
.put_w64:
    movu                 m0, [srcq+ssq*0]
    movu                 m1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    mova       [dstq+dsq*0], m0
    mova       [dstq+dsq*1], m1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w64
    RET
.put_w128:
    movu                 m0, [srcq+ssq*0+64*0]
    movu                 m1, [srcq+ssq*0+64*1]
    movu                 m2, [srcq+ssq*1+64*0]
    movu                 m3, [srcq+ssq*1+64*1]
    lea                srcq, [srcq+ssq*2]
    mova  [dstq+dsq*0+64*0], m0
    mova  [dstq+dsq*0+64*1], m1
    mova  [dstq+dsq*1+64*0], m2
    mova  [dstq+dsq*1+64*1], m3
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .put_w128
    RET
.h:
    ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4
    ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4
    imul               mxyd, 0xff01
    vbroadcasti128       m4, [bilin_h_shuf8]
    add                mxyd, 16 << 8
    vpbroadcastw         m5, mxyd
    mov                mxyd, r7m ; my
    test               mxyd, mxyd
    jnz .hv
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_h)]
    vpbroadcastd         m3, [pw_2048]
    add                  wq, r7
    jmp                  wq
.h_w2:
    movd               xmm0, [srcq+ssq*0]
    pinsrd             xmm0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pshufb             xmm0, xm4
    pmaddubsw          xmm0, xm5
    pmulhrsw           xmm0, xm3
    packuswb           xmm0, xmm0
    pextrw     [dstq+dsq*0], xmm0, 0
    pextrw     [dstq+dsq*1], xmm0, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w2
    RET
.h_w4:
    mova               xmm4, [bilin_h_shuf4]
.h_w4_loop:
    movq               xmm0, [srcq+ssq*0]
    movhps             xmm0, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb             xmm0, xmm4
    pmaddubsw          xmm0, xm5
    pmulhrsw           xmm0, xm3
    packuswb           xmm0, xmm0
    movd       [dstq+dsq*0], xmm0
    pextrd     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w4_loop
    RET
.h_w8:
    movu                xm0, [srcq+ssq*0]
    vinserti32x4        ym0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pshufb              ym0, ym4
    pmaddubsw           ym0, ym5
    pmulhrsw            ym0, ym3
    vpmovuswb           xm0, ym0
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    mova                 m4, [bilin_h_perm16]
.h_w16_loop:
    movu                ym0, [srcq+ssq*0]
    vinserti32x8         m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    vpermb               m0, m4, m0
    pmaddubsw            m0, m5
    pmulhrsw             m0, m3
    vpmovuswb           ym0, m0
    mova         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], ym0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w16_loop
    RET
.h_w32:
    movu                ym0, [srcq+ssq*0+8*0]
    vinserti32x8         m0, [srcq+ssq*1+8*0], 1
    movu                ym1, [srcq+ssq*0+8*1]
    vinserti32x8         m1, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmulhrsw             m0, m3
    pmulhrsw             m1, m3
    packuswb             m0, m1
    mova          [dstq+dsq*0], ym0
    vextracti32x8 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w32
    RET
.h_w64:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmulhrsw             m0, m3
    pmulhrsw             m1, m3
    packuswb             m0, m1
    add                srcq, ssq
    mova             [dstq], m0
    add                dstq, dsq
    dec                  hd
    jg .h_w64
    RET
.h_w128:
    movu                 m0, [srcq+8*0]
    movu                 m2, [srcq+8*1]
    movu                 m1, [srcq+8*8]
    movu                 m6, [srcq+8*9]
    add                srcq, ssq
    REPX  {pshufb    x, m4}, m0, m2, m1, m6
    REPX  {pmaddubsw x, m5}, m0, m2, m1, m6
    REPX  {pmulhrsw  x, m3}, m0, m2, m1, m6
    packuswb             m0, m2
    packuswb             m1, m6
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m1
    add                dstq, dsq
    dec                  hd
    jg .h_w128
    RET
.v:
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_v)]
    imul               mxyd, 0xff01
    vpbroadcastd         m5, [pw_2048]
    add                mxyd, 16 << 8
    add                  wq, r7
    vpbroadcastw         m4, mxyd
    jmp                  wq
.v_w2:
    movd               xmm0,       [srcq+ssq*0]
.v_w2_loop:
    pinsrw             xmm1, xmm0, [srcq+ssq*1], 1 ; 0 1
    lea                srcq,       [srcq+ssq*2]
    pinsrw             xmm0, xmm1, [srcq+ssq*0], 0 ; 2 1
    pshuflw            xmm1, xmm1, q2301           ; 1 0
    punpcklbw          xmm1, xmm0, xmm1
    pmaddubsw          xmm1, xm4
    pmulhrsw           xmm1, xm5
    packuswb           xmm1, xmm1
    pextrw     [dstq+dsq*0], xmm1, 1
    pextrw     [dstq+dsq*1], xmm1, 0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w2_loop
    RET
.v_w4:
    movd               xmm0, [srcq+ssq*0]
.v_w4_loop:
    vpbroadcastd       xmm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd           xmm2, xmm1, xmm0, 0x01 ; 0 1
    vpbroadcastd       xmm0, [srcq+ssq*0]
    vpblendd           xmm1, xmm0, 0x02       ; 1 2
    punpcklbw          xmm1, xmm2
    pmaddubsw          xmm1, xm4
    pmulhrsw           xmm1, xm5
    packuswb           xmm1, xmm1
    movd       [dstq+dsq*0], xmm1
    pextrd     [dstq+dsq*1], xmm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w4_loop
    RET
.v_w8:
    movq               xmm0, [srcq+ssq*0]
.v_w8_loop:
    movq               xmm3, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    punpcklbw          xmm1, xmm3, xmm0
    movq               xmm0, [srcq+ssq*0]
    punpcklbw          xmm2, xmm0, xmm3
    pmaddubsw          xmm1, xm4
    pmaddubsw          xmm2, xm4
    pmulhrsw           xmm1, xm5
    pmulhrsw           xmm2, xm5
    packuswb           xmm1, xmm2
    movq       [dstq+dsq*0], xmm1
    movhps     [dstq+dsq*1], xmm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w8_loop
    RET
.v_w16:
    movu               xmm0, [srcq+ssq*0]
.v_w16_loop:
    vbroadcasti128     ymm2, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd           ymm3, ymm2, ymm0, 0x0f ; 0 1
    vbroadcasti128     ymm0, [srcq+ssq*0]
    vpblendd           ymm2, ymm2, ymm0, 0xf0 ; 1 2
    punpcklbw          ymm1, ymm2, ymm3
    punpckhbw          ymm2, ymm3
    pmaddubsw          ymm1, ym4
    pmaddubsw          ymm2, ym4
    pmulhrsw           ymm1, ym5
    pmulhrsw           ymm2, ym5
    packuswb           ymm1, ymm2
    mova         [dstq+dsq*0], xmm1
    vextracti128 [dstq+dsq*1], ymm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w16_loop
    vzeroupper
    RET
.v_w32:
    movu                ym0, [srcq+ssq*0]
    kxnorb               k1, k1, k1
.v_w32_loop:
    vbroadcasti32x8      m2, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendmd        m3{k1}, m2, m0 ; 0 1
    vbroadcasti32x8      m0, [srcq+ssq*0]
    vpblendmd        m2{k1}, m0, m2 ; 1 2
    punpcklbw            m1, m2, m3
    punpckhbw            m2, m3
    pmaddubsw            m1, m4
    pmaddubsw            m2, m4
    pmulhrsw             m1, m5
    pmulhrsw             m2, m5
    packuswb             m1, m2
    mova          [dstq+dsq*0], ym1
    vextracti32x8 [dstq+dsq*1], m1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w32_loop
    RET
.v_w64:
    movu                 m0, [srcq+ssq*0]
.v_w64_loop:
    movu                 m3, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    punpcklbw            m1, m3, m0
    punpckhbw            m6, m3, m0
    movu                 m0, [srcq+ssq*0]
    pmaddubsw            m1, m4
    pmaddubsw            m6, m4
    punpcklbw            m2, m0, m3
    punpckhbw            m7, m0, m3
    pmaddubsw            m2, m4
    pmaddubsw            m7, m4
    REPX   {pmulhrsw x, m5}, m1, m6, m2, m7
    packuswb             m1, m6
    packuswb             m2, m7
    mova       [dstq+dsq*0], m1
    mova       [dstq+dsq*1], m2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w64_loop
    RET
.v_w128:
    movu                 m0, [srcq+64*0]
    movu                 m1, [srcq+64*1]
.v_w128_loop:
    add                srcq, ssq
    movu                 m2, [srcq+64*0]
    movu                 m3, [srcq+64*1]
    punpcklbw            m6, m2, m0
    pmaddubsw            m6, m4
    punpckhbw            m0, m2, m0
    pmaddubsw            m0, m4
    punpcklbw            m7, m3, m1
    pmaddubsw            m7, m4
    punpckhbw            m1, m3, m1
    pmaddubsw            m1, m4
    REPX   {pmulhrsw x, m5}, m6, m0, m7, m1
    packuswb             m6, m0
    mova                 m0, m2
    packuswb             m7, m1
    mova                 m1, m3
    mova        [dstq+64*0], m6
    mova        [dstq+64*1], m7
    add                dstq, dsq
    dec                  hd
    jg .v_w128_loop
    RET
.hv:
    ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8
    ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4
    movzx                wd, word [r7+wq*2+table_offset(put, _bilin_hv)]
    WIN64_SPILL_XMM       8
    shl                mxyd, 11 ; can't shift by 12 due to signed overflow
    vpbroadcastd         m7, [pw_2048]
    add                  wq, r7
    vpbroadcastw         m6, mxyd
    jmp                  wq
.hv_w2:
    vpbroadcastd       xmm0, [srcq+ssq*0]
    pshufb             xmm0, xm4
    pmaddubsw          xmm0, xm5
.hv_w2_loop:
    movd               xmm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pinsrd             xmm1, [srcq+ssq*0], 1
    pshufb             xmm1, xm4
    pmaddubsw          xmm1, xm5               ; 1 _ 2 _
    shufps             xmm2, xmm0, xmm1, q1032 ; 0 _ 1 _
    mova               xmm0, xmm1
    psubw              xmm1, xmm2
    paddw              xmm1, xmm1
    pmulhw             xmm1, xm6
    paddw              xmm1, xmm2
    pmulhrsw           xmm1, xm7
    packuswb           xmm1, xmm1
    pextrw     [dstq+dsq*0], xmm1, 0
    pextrw     [dstq+dsq*1], xmm1, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w2_loop
    RET
.hv_w4:
    mova               xmm4, [bilin_h_shuf4]
    movddup            xmm0, [srcq+ssq*0]
    pshufb             xmm0, xmm4
    pmaddubsw          xmm0, xm5
.hv_w4_loop:
    movq               xmm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movhps             xmm1, [srcq+ssq*0]
    pshufb             xmm1, xmm4
    pmaddubsw          xmm1, xm5               ; 1 2
    shufps             xmm2, xmm0, xmm1, q1032 ; 0 1
    mova               xmm0, xmm1
    psubw              xmm1, xmm2
    paddw              xmm1, xmm1
    pmulhw             xmm1, xm6
    paddw              xmm1, xmm2
    pmulhrsw           xmm1, xm7
    packuswb           xmm1, xmm1
    movd       [dstq+dsq*0], xmm1
    pextrd     [dstq+dsq*1], xmm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w4_loop
    RET
.hv_w8:
    vbroadcasti128      ym0, [srcq+ssq*0]
    pshufb              ym0, ym4
    pmaddubsw           ym0, ym5
.hv_w8_loop:
    movu                xm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vinserti128         ym1, [srcq+ssq*0], 1
    pshufb              ym1, ym4
    pmaddubsw           ym1, ym5            ; 1 2
    valignq             ym2, ym1, ym0, 2
    mova                ym0, ym1
    psubw               ym1, ym2
    paddw               ym1, ym1
    pmulhw              ym1, ym6
    paddw               ym1, ym2
    pmulhrsw            ym1, ym7
    vpmovuswb           xm1, ym1
    movq       [dstq+dsq*0], xm1
    movhps     [dstq+dsq*1], xm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w8_loop
    RET
.hv_w16:
    vbroadcasti32x8      m0, [srcq+ssq*0]
    mova                 m4, [bilin_h_perm16]
    vpermb               m0, m4, m0
    pmaddubsw            m0, m5
.hv_w16_loop:
    movu                ym1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vinserti32x8         m1, [srcq+ssq*0], 1
    vpermb               m1, m4, m1
    pmaddubsw            m1, m5        ; 1 2
    valignq              m2, m1, m0, 4 ; 0 1
    mova                 m0, m1
    psubw                m1, m2
    paddw                m1, m1
    pmulhw               m1, m6
    paddw                m1, m2
    pmulhrsw             m1, m7
    vpmovuswb           ym1, m1
    mova          [dstq+dsq*0], xm1
    vextracti32x4 [dstq+dsq*1], ym1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w16_loop
    RET
.hv_w32:
    mova                 m4, [bilin_h_perm32]
    vpermb               m0, m4, [srcq+ssq*0]
    pmovzxbq             m8, [pb_02461357]
    pmaddubsw            m0, m5
.hv_w32_loop:
    vpermb               m2, m4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpermb               m3, m4, [srcq+ssq*0]
    pmaddubsw            m2, m5
    psubw                m1, m2, m0
    paddw                m1, m1
    pmulhw               m1, m6
    paddw                m1, m0
    pmaddubsw            m0, m3, m5
    psubw                m3, m0, m2
    paddw                m3, m3
    pmulhw               m3, m6
    paddw                m3, m2
    pmulhrsw             m1, m7
    pmulhrsw             m3, m7
    packuswb             m1, m3
    vpermq               m1, m8, m1
    mova          [dstq+dsq*0], ym1
    vextracti32x8 [dstq+dsq*1], m1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w32_loop
    RET
.hv_w64:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    pshufb               m0, m4
    pshufb               m1, m4
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
.hv_w64_loop:
    add                srcq, ssq
    movu                 m2, [srcq+8*0]
    movu                 m3, [srcq+8*1]
    pshufb               m2, m4
    pshufb               m3, m4
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    psubw                m8, m2, m0
    psubw                m9, m3, m1
    paddw                m8, m8
    pmulhw               m8, m6
    paddw                m9, m9
    pmulhw               m9, m6
    paddw                m8, m0
    pmulhrsw             m8, m7
    paddw                m9, m1
    pmulhrsw             m9, m7
    mova                 m0, m2
    mova                 m1, m3
    packuswb             m8, m9
    mova             [dstq], m8
    add                dstq, dsq
    dec                  hd
    jg .hv_w64_loop
    RET
.hv_w128:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    movu                 m2, [srcq+8*8]
    movu                 m3, [srcq+8*9]
    REPX  {pshufb    x, m4}, m0, m1, m2, m3
    REPX  {pmaddubsw x, m5}, m0, m1, m2, m3
.hv_w128_loop:
    add                srcq, ssq
    movu                 m8, [srcq+8*0]
    movu                 m9, [srcq+8*1]
    movu                m10, [srcq+8*8]
    movu                m11, [srcq+8*9]
    REPX  {pshufb    x, m4}, m8, m9, m10, m11
    REPX  {pmaddubsw x, m5}, m8, m9, m10, m11
    psubw               m12, m8, m0
    psubw               m13, m9, m1
    psubw               m14, m10, m2
    psubw               m15, m11, m3
    paddw               m12, m12
    pmulhw              m12, m6
    paddw               m13, m13
    pmulhw              m13, m6
    paddw               m14, m14
    pmulhw              m14, m6
    paddw               m15, m15
    pmulhw              m15, m6
    paddw               m12, m0
    pmulhrsw            m12, m7
    paddw               m13, m1
    pmulhrsw            m13, m7
    paddw               m14, m2
    pmulhrsw            m14, m7
    paddw               m15, m3
    pmulhrsw            m15, m7
    mova                 m0, m8
    mova                 m1, m9
    mova                 m2, m10
    mova                 m3, m11
    packuswb            m12, m13
    packuswb            m14, m15
    mova        [dstq+64*0], m12
    mova        [dstq+64*1], m14
    add                dstq, dsq
    dec                  hd
    jg .hv_w128_loop
    RET

DECLARE_REG_TMP 3, 5, 6

cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3
    movifnidn          mxyd, r5m ; mx
    lea                  t2, [prep_avx512icl]
    tzcnt                wd, wm
    movifnidn            hd, hm
    test               mxyd, mxyd
    jnz .h
    mov                mxyd, r6m ; my
    test               mxyd, mxyd
    jnz .v
.prep:
    movzx                wd, word [t2+wq*2+table_offset(prep,)]
    add                  wq, t2
    lea            stride3q, [strideq*3]
    jmp                  wq
.prep_w4:
    movd               xmm0, [srcq+strideq*0]
    pinsrd             xmm0, [srcq+strideq*1], 1
    pinsrd             xmm0, [srcq+strideq*2], 2
    pinsrd             xmm0, [srcq+stride3q ], 3
    lea                srcq, [srcq+strideq*4]
    pmovzxbw            ym0, xmm0
    psllw               ym0, 4
    mova             [tmpq], ym0
    add                tmpq, 32
    sub                  hd, 4
    jg .prep_w4
    RET
.prep_w8:
    movq               xmm0, [srcq+strideq*0]
    movq               xmm1, [srcq+strideq*1]
    vinserti128         ym0, ymm0, [srcq+strideq*2], 1
    vinserti128         ym1, ymm1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    punpcklqdq          ym0, ym1
    pmovzxbw             m0, ym0
    psllw                m0, 4
    mova             [tmpq], m0
    add                tmpq, 32*2
    sub                  hd, 4
    jg .prep_w8
    RET
.prep_w16:
    movu               xmm0, [srcq+strideq*0]
    vinserti128         ym0, ymm0, [srcq+strideq*1], 1
    movu               xmm1, [srcq+strideq*2]
    vinserti128         ym1, ymm1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    pmovzxbw             m0, ym0
    pmovzxbw             m1, ym1
    psllw                m0, 4
    psllw                m1, 4
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    add                tmpq, 32*4
    sub                  hd, 4
    jg .prep_w16
    RET
.prep_w32:
    pmovzxbw             m0, [srcq+strideq*0]
    pmovzxbw             m1, [srcq+strideq*1]
    pmovzxbw             m2, [srcq+strideq*2]
    pmovzxbw             m3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    mova        [tmpq+64*2], m2
    mova        [tmpq+64*3], m3
    add                tmpq, 64*4
    sub                  hd, 4
    jg .prep_w32
    RET
.prep_w64:
    pmovzxbw             m0, [srcq+strideq*0+32*0]
    pmovzxbw             m1, [srcq+strideq*0+32*1]
    pmovzxbw             m2, [srcq+strideq*1+32*0]
    pmovzxbw             m3, [srcq+strideq*1+32*1]
    lea                srcq, [srcq+strideq*2]
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    mova        [tmpq+64*2], m2
    mova        [tmpq+64*3], m3
    add                tmpq, 64*4
    sub                  hd, 2
    jg .prep_w64
    RET
.prep_w128:
    pmovzxbw             m0, [srcq+32*0]
    pmovzxbw             m1, [srcq+32*1]
    pmovzxbw             m2, [srcq+32*2]
    pmovzxbw             m3, [srcq+32*3]
    REPX       {psllw x, 4}, m0, m1, m2, m3
    mova    [tmpq+64*0], m0
    mova    [tmpq+64*1], m1
    mova    [tmpq+64*2], m2
    mova    [tmpq+64*3], m3
    add                tmpq, 64*4
    add                srcq, strideq
    dec                  hd
    jg .prep_w128
    RET
.h:
    ; 16 * src[x] + (mx * (src[x + 1] - src[x]))
    ; = (16 - mx) * src[x] + mx * src[x + 1]
    imul               mxyd, 0xff01
    add                mxyd, 16 << 8
    vpbroadcastw         m5, mxyd
    mov                mxyd, r6m ; my
    test               mxyd, mxyd
    jnz .hv
    movzx                wd, word [t2+wq*2+table_offset(prep, _bilin_h)]
    add                  wq, t2
    lea            stride3q, [strideq*3]
    jmp                  wq
.h_w4:
    vbroadcasti32x4     ym4, [bilin_h_shuf4]
.h_w4_loop:
    movq               xmm0, [srcq+strideq*0]
    movq               xmm1, [srcq+strideq*1]
    vinserti32x4        ym0, ymm0, [srcq+strideq*2], 1
    vinserti32x4        ym1, ymm1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    punpcklqdq          ym0, ym1
    pshufb              ym0, ym4
    pmaddubsw           ym0, ym5
    mova             [tmpq], ym0
    add                tmpq, 32
    sub                  hd, 4
    jg .h_w4_loop
    RET
.h_w8:
    vbroadcasti32x4      m4, [bilin_h_shuf8]
.h_w8_loop:
    movu               xmm0, [srcq+strideq*0]
    vinserti32x4        ym0, ymm0, [srcq+strideq*1], 1
    vinserti32x4         m0, [srcq+strideq*2], 2
    vinserti32x4         m0, [srcq+stride3q ], 3
    lea                srcq, [srcq+strideq*4]
    pshufb               m0, m4
    pmaddubsw            m0, m5
    mova             [tmpq], m0
    add                tmpq, 64
    sub                  hd, 4
    jg .h_w8_loop
    RET
.h_w16:
    mova                 m4, [bilin_h_perm16]
.h_w16_loop:
    movu                ym0, [srcq+strideq*0]
    vinserti32x8         m0, [srcq+strideq*1], 1
    movu                ym1, [srcq+strideq*2]
    vinserti32x8         m1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    vpermb               m0, m4, m0
    vpermb               m1, m4, m1
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    add                tmpq, 64*2
    sub                  hd, 4
    jg .h_w16_loop
    RET
.h_w32:
    mova                 m4, [bilin_h_perm32]
.h_w32_loop:
    vpermb               m0, m4, [srcq+strideq*0]
    vpermb               m1, m4, [srcq+strideq*1]
    vpermb               m2, m4, [srcq+strideq*2]
    vpermb               m3, m4, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    mova        [tmpq+64*2], m2
    mova        [tmpq+64*3], m3
    add                tmpq, 64*4
    sub                  hd, 4
    jg .h_w32_loop
    RET
.h_w64:
    mova                 m4, [bilin_h_perm32]
.h_w64_loop:
    vpermb               m0, m4, [srcq+strideq*0+32*0]
    vpermb               m1, m4, [srcq+strideq*0+32*1]
    vpermb               m2, m4, [srcq+strideq*1+32*0]
    vpermb               m3, m4, [srcq+strideq*1+32*1]
    lea                srcq, [srcq+strideq*2]
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    mova        [tmpq+64*2], m2
    mova        [tmpq+64*3], m3
    add                tmpq, 64*4
    sub                  hd, 2
    jg .h_w64_loop
    RET
.h_w128:
    mova                 m4, [bilin_h_perm32]
.h_w128_loop:
    vpermb               m0, m4, [srcq+32*0]
    vpermb               m1, m4, [srcq+32*1]
    vpermb               m2, m4, [srcq+32*2]
    vpermb               m3, m4, [srcq+32*3]
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
    mova        [tmpq+64*2], m2
    mova        [tmpq+64*3], m3
    add                tmpq, 64*4
    add                srcq, strideq
    dec                  hd
    jg .h_w128_loop
    RET
.v:
    WIN64_SPILL_XMM       7
    movzx                wd, word [t2+wq*2+table_offset(prep, _bilin_v)]
    imul               mxyd, 0xff01
    add                mxyd, 16 << 8
    add                  wq, t2
    lea            stride3q, [strideq*3]
    vpbroadcastw         m6, mxyd
    jmp                  wq
.v_w4:
    vpbroadcastd        xm0, [srcq+strideq*0]
    mov                 r3d, 0x29
    vbroadcasti32x4     ym3, [bilin_v_shuf4]
    kmovb                k1, r3d
.v_w4_loop:
    vpblendmd       xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____
    vpbroadcastd        ym2, [srcq+strideq*2]
    vpbroadcastd    ym2{k1}, [srcq+stride3q ]             ; __2_ 23__
    lea                srcq, [srcq+strideq*4]
    vpbroadcastd        ym0, [srcq+strideq*0]
    punpckhqdq      ym2{k1}, ym1, ym0                     ; 012_ 234_
    pshufb              ym2, ym3
    pmaddubsw           ym2, ym6
    mova             [tmpq], ym2
    add                tmpq, 32
    sub                  hd, 4
    jg .v_w4_loop
    RET
.v_w8:
    mova                 m5, [bilin_v_perm8]
    vbroadcasti32x4     ym0, [srcq+strideq*0]
.v_w8_loop:
    vinserti32x4        ym1, ym0, [srcq+strideq*1], 1
    vpbroadcastq        ym0, [srcq+strideq*2]
    vinserti32x4         m1, [srcq+stride3q ], 2
    lea                srcq, [srcq+strideq*4]
    vinserti32x4        ym0, [srcq+strideq*0], 0
    vpermt2b             m1, m5, m0
    pmaddubsw            m1, m6
    mova             [tmpq], m1
    add                tmpq, 64
    sub                  hd, 4
    jg .v_w8_loop
    RET
.v_w16:
    mova                 m5, [bilin_v_perm16]
    movu                xm0, [srcq+strideq*0]
.v_w16_loop:
    movu                xm2, [srcq+strideq*2]
    vinserti32x4        ym1, ym0, [srcq+strideq*1], 1
    vpermt2b             m1, m5, m2
    vinserti32x4        ym2, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    movu                xm0, [srcq+strideq*0]
    vpermt2b             m2, m5, m0
    pmaddubsw            m1, m6
    pmaddubsw            m2, m6
    mova        [tmpq+64*0], m1
    mova        [tmpq+64*1], m2
    add                tmpq, 64*2
    sub                  hd, 4
    jg .v_w16_loop
    RET
.v_w32:
    mova                 m5, [bilin_v_perm32]
    movu                ym0, [srcq+strideq*0]
.v_w32_loop:
    movu                ym2, [srcq+strideq*1]
    movu                ym3, [srcq+strideq*2]
    movu                ym4, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vpermt2b             m0, m5, m2
    vpermt2b             m2, m5, m3
    vpermt2b             m3, m5, m4
    pmaddubsw            m1, m0, m6
    movu                ym0, [srcq+strideq*0]
    vpermt2b             m4, m5, m0
    pmaddubsw            m2, m6
    pmaddubsw            m3, m6
    pmaddubsw            m4, m6
    mova        [tmpq+64*0], m1
    mova        [tmpq+64*1], m2
    mova        [tmpq+64*2], m3
    mova        [tmpq+64*3], m4
    add                tmpq, 64*4
    sub                  hd, 4
    jg .v_w32_loop
    RET
.v_w64:
    mova                 m5, [bilin_v_perm64]
    vpermq               m0, m5, [srcq+strideq*0]
.v_w64_loop:
    vpermq               m1, m5, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    punpcklbw            m4, m1, m0
    punpckhbw            m2, m1, m0
    vpermq               m0, m5, [srcq+strideq*0]
    punpcklbw            m3, m0, m1
    punpckhbw            m1, m0, m1
    pmaddubsw            m4, m6
    pmaddubsw            m2, m6
    pmaddubsw            m3, m6
    pmaddubsw            m1, m6
    mova        [tmpq+64*0], m4
    mova        [tmpq+64*1], m2
    mova        [tmpq+64*2], m3
    mova        [tmpq+64*3], m1
    add                tmpq, 64*4
    sub                  hd, 2
    jg .v_w64_loop
    RET
.v_w128:
    mova                 m5, [bilin_v_perm64]
    vpermq               m0, m5, [srcq+strideq*0+ 0]
    vpermq               m1, m5, [srcq+strideq*0+64]
.v_w128_loop:
    vpermq               m2, m5, [srcq+strideq*1+ 0]
    vpermq               m3, m5, [srcq+strideq*1+64]
    lea                srcq, [srcq+strideq*2]
    punpcklbw            m4, m2, m0
    punpckhbw            m0, m2, m0
    pmaddubsw            m4, m6
    pmaddubsw            m0, m6
    mova        [tmpq+64*0], m4
    mova        [tmpq+64*1], m0
    punpcklbw            m4, m3, m1
    punpckhbw            m1, m3, m1
    pmaddubsw            m4, m6
    pmaddubsw            m1, m6
    mova        [tmpq+64*2], m4
    mova        [tmpq+64*3], m1
    vpermq               m0, m5, [srcq+strideq*0+ 0]
    vpermq               m1, m5, [srcq+strideq*0+64]
    punpcklbw            m4, m0, m2
    punpckhbw            m2, m0, m2
    pmaddubsw            m4, m6
    pmaddubsw            m2, m6
    mova        [tmpq+64*4], m4
    mova        [tmpq+64*5], m2
    punpcklbw            m4, m1, m3
    punpckhbw            m3, m1, m3
    pmaddubsw            m4, m6
    pmaddubsw            m3, m6
    mova        [tmpq+64*6], m4
    mova        [tmpq+64*7], m3
    add                tmpq, 64*8
    sub                  hd, 2
    jg .v_w128_loop
    RET
.hv:
    ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4
    ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4)
    %assign stack_offset stack_offset - stack_size_padded
    WIN64_SPILL_XMM       7
    movzx                wd, word [t2+wq*2+table_offset(prep, _bilin_hv)]
    shl                mxyd, 11
    vpbroadcastw         m6, mxyd
    add                  wq, t2
    lea            stride3q, [strideq*3]
    jmp                  wq
.hv_w4:
    vbroadcasti32x4     ym4, [bilin_h_shuf4]
    vpbroadcastq        ym0, [srcq+strideq*0]
    pshufb              ym0, ym4
    pmaddubsw           ym0, ym5
.hv_w4_loop:
    movq               xmm1, [srcq+strideq*1]
    movq               xmm2, [srcq+strideq*2]
    vinserti32x4        ym1, ymm1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    vinserti32x4        ym2, ymm2, [srcq+strideq*0], 1
    punpcklqdq          ym1, ym2
    pshufb              ym1, ym4
    pmaddubsw           ym1, ym5         ; 1 2 3 4
    valignq             ym2, ym1, ym0, 3 ; 0 1 2 3
    mova                ym0, ym1
    psubw               ym1, ym2
    pmulhrsw            ym1, ym6
    paddw               ym1, ym2
    mova             [tmpq], ym1
    add                tmpq, 32
    sub                  hd, 4
    jg .hv_w4_loop
    RET
.hv_w8:
    vbroadcasti32x4      m4, [bilin_h_shuf8]
    vbroadcasti32x4      m0, [srcq+strideq*0]
    pshufb               m0, m4
    pmaddubsw            m0, m5
.hv_w8_loop:
    movu               xmm1, [srcq+strideq*1]
    vinserti128         ym1, ymm1, [srcq+strideq*2], 1
    vinserti128          m1, [srcq+stride3q ], 2
    lea                srcq, [srcq+strideq*4]
    vinserti128          m1, [srcq+strideq*0], 3
    pshufb               m1, m4
    pmaddubsw            m1, m5        ; 1 2 3 4
    valignq              m2, m1, m0, 6 ; 0 1 2 3
    mova                 m0, m1
    psubw                m1, m2
    pmulhrsw             m1, m6
    paddw                m1, m2
    mova             [tmpq], m1
    add                tmpq, 64
    sub                  hd, 4
    jg .hv_w8_loop
    RET
.hv_w16:
    mova                 m4, [bilin_h_perm16]
    vbroadcasti32x8      m0, [srcq+strideq*0]
    vpermb               m0, m4, m0
    pmaddubsw            m0, m5
.hv_w16_loop:
    movu                ym1, [srcq+strideq*1]
    vinserti32x8         m1, [srcq+strideq*2], 1
    movu                ym2, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vinserti32x8         m2, [srcq+strideq*0], 1
    vpermb               m1, m4, m1
    vpermb               m2, m4, m2
    pmaddubsw            m1, m5            ; 1 2
    vshufi32x4           m3, m0, m1, q1032 ; 0 1
    pmaddubsw            m0, m2, m5        ; 3 4
    vshufi32x4           m2, m1, m0, q1032 ; 2 3
    psubw                m1, m3
    pmulhrsw             m1, m6
    paddw                m1, m3
    psubw                m3, m0, m2
    pmulhrsw             m3, m6
    paddw                m3, m2
    mova        [tmpq+64*0], m1
    mova        [tmpq+64*1], m3
    add                tmpq, 64*2
    sub                  hd, 4
    jg .hv_w16_loop
    RET
.hv_w32:
    mova                 m4, [bilin_h_perm32]
    vpermb               m0, m4, [srcq+strideq*0]
    pmaddubsw            m0, m5
.hv_w32_loop:
    vpermb               m1, m4, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    vpermb               m2, m4, [srcq+strideq*0]
    pmaddubsw            m1, m5
    psubw                m3, m1, m0
    pmulhrsw             m3, m6
    paddw                m3, m0
    pmaddubsw            m0, m2, m5
    psubw                m2, m0, m1
    pmulhrsw             m2, m6
    paddw                m2, m1
    mova        [tmpq+64*0], m3
    mova        [tmpq+64*1], m2
    add                tmpq, 64*2
    sub                  hd, 2
    jg .hv_w32_loop
    RET
.hv_w64:
    mova                 m4, [bilin_h_perm32]
    vpermb               m0, m4, [srcq+32*0]
    vpermb               m1, m4, [srcq+32*1]
    pmaddubsw            m0, m5
    pmaddubsw            m1, m5
.hv_w64_loop:
    add                srcq, strideq
    vpermb               m2, m4, [srcq+32*0]
    vpermb               m3, m4, [srcq+32*1]
    pmaddubsw            m2, m5
    pmaddubsw            m3, m5
    psubw                m7, m2, m0
    psubw                m8, m3, m1
    pmulhrsw             m7, m6
    pmulhrsw             m8, m6
    paddw                m7, m0
    mova                 m0, m2
    paddw                m8, m1
    mova                 m1, m3
    mova        [tmpq+64*0], m7
    mova        [tmpq+64*1], m8
    add                tmpq, 64*2
    dec                  hd
    jg .hv_w64_loop
    RET
.hv_w128:
    mova                 m4, [bilin_h_perm32]
    vpermb               m0, m4, [srcq+32*0]
    vpermb               m1, m4, [srcq+32*1]
    vpermb               m2, m4, [srcq+32*2]
    vpermb               m3, m4, [srcq+32*3]
    REPX  {pmaddubsw x, m5}, m0, m1, m2, m3
.hv_w128_loop:
    add                srcq, strideq
    vpermb               m7, m4, [srcq+32*0]
    vpermb               m8, m4, [srcq+32*1]
    vpermb               m9, m4, [srcq+32*2]
    vpermb              m10, m4, [srcq+32*3]
    REPX  {pmaddubsw x, m5}, m7, m8, m9, m10
    psubw               m11, m7, m0
    psubw               m12, m8, m1
    psubw               m13, m9, m2
    psubw               m14, m10, m3
    REPX  {pmulhrsw  x, m6}, m11, m12, m13, m14
    paddw               m11, m0
    mova                 m0, m7
    paddw               m12, m1
    mova                 m1, m8
    paddw               m13, m2
    mova                 m2, m9
    paddw               m14, m3
    mova                 m3, m10
    mova        [tmpq+64*0], m11
    mova        [tmpq+64*1], m12
    mova        [tmpq+64*2], m13
    mova        [tmpq+64*3], m14
    add                tmpq, 64*4
    dec                  hd
    jg .hv_w128_loop
    RET

; int8_t subpel_filters[5][15][8]
%assign FILTER_REGULAR (0*15 << 16) | 3*15
%assign FILTER_SMOOTH  (1*15 << 16) | 4*15
%assign FILTER_SHARP   (2*15 << 16) | 3*15

%macro FN 4 ; fn, type, type_h, type_v
cglobal %1_%2_8bpc
    mov                 t0d, FILTER_%3
%ifidn %3, %4
    mov                 t1d, t0d
%else
    mov                 t1d, FILTER_%4
%endif
%ifnidn %2, regular ; skip the jump in the last filter
    jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX)
%endif
%endmacro

%macro PUT_8TAP_H 4-5 0 ; dst/src, tmp[1-3], vpermb
%if %5
    vpermb              m%2, m6, m%1
    vpermb              m%3, m7, m%1
    vpermb              m%4, m8, m%1
%else
%if %2 < %4 ; reuse a previous value if possible
    pshufb              m%2, m%1, m6
%endif
    pshufb              m%3, m%1, m7
    pshufb              m%4, m%1, m8
%endif
    mova                m%1, m5
    vpdpbusd            m%1, m%2, m9
    mova                m%2, m5
    vpdpbusd            m%2, m%3, m9
    vpdpbusd            m%1, m%3, m10
    vpdpbusd            m%2, m%4, m10
    packusdw            m%1, m%2
    psrlw               m%1, 6
%endmacro

%if WIN64
DECLARE_REG_TMP 4, 5
%else
DECLARE_REG_TMP 7, 8
%endif

%define PUT_8TAP_FN FN put_8tap,

PUT_8TAP_FN sharp,          SHARP,   SHARP
PUT_8TAP_FN sharp_smooth,   SHARP,   SMOOTH
PUT_8TAP_FN smooth_sharp,   SMOOTH,  SHARP
PUT_8TAP_FN smooth,         SMOOTH,  SMOOTH
PUT_8TAP_FN sharp_regular,  SHARP,   REGULAR
PUT_8TAP_FN regular_sharp,  REGULAR, SHARP
PUT_8TAP_FN smooth_regular, SMOOTH,  REGULAR
PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH
PUT_8TAP_FN regular,        REGULAR, REGULAR

cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3
%define base r8-put_avx512icl
    imul                mxd, mxm, 0x010101
    add                 mxd, t0d ; 8tap_h, mx, 4tap_h
    imul                myd, mym, 0x010101
    add                 myd, t1d ; 8tap_v, my, 4tap_v
    lea                  r8, [put_avx512icl]
    movsxd               wq, wm
    movifnidn            hd, hm
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jnz .v
    tzcnt                wd, wd
    movzx                wd, word [r8+wq*2+table_offset(put,)]
    add                  wq, r8
    lea                  r6, [ssq*3]
    lea                  r7, [dsq*3]
%if WIN64
    pop                  r8
%endif
    jmp                  wq
.h:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m5, [pd_34] ; 2 + (8 << 2)
    WIN64_SPILL_XMM      11
    cmp                  wd, 4
    jl .h_w2
    vbroadcasti128       m6, [subpel_h_shufA]
    je .h_w4
    tzcnt                wd, wd
    vbroadcasti128       m7, [subpel_h_shufB]
    vbroadcasti128       m8, [subpel_h_shufC]
    shr                 mxd, 16
    sub                srcq, 3
    movzx                wd, word [r8+wq*2+table_offset(put, _8tap_h)]
    vpbroadcastd         m9, [base+mxq*8+subpel_filters+0]
    vpbroadcastd        m10, [base+mxq*8+subpel_filters+4]
    add                  wq, r8
    jmp                  wq
.h_w2:
    movzx               mxd, mxb
    dec                srcq
    mova               xmm4, [subpel_h_shuf4]
    vpbroadcastd       xmm3, [base+mxq*8+subpel_filters+2]
.h_w2_loop:
    movq               xmm0, [srcq+ssq*0]
    movhps             xmm0, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb             xmm0, xmm4
    mova               xmm1, xm5
    vpdpbusd           xmm1, xmm0, xmm3
    packssdw           xmm0, xmm1, xmm1
    psraw              xmm0, 6
    packuswb           xmm0, xm0
    pextrw     [dstq+dsq*0], xmm0, 0
    pextrw     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w2_loop
    RET
.h_w4:
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd       xmm3, [base+mxq*8+subpel_filters+2]
.h_w4_loop:
    movq               xmm0, [srcq+ssq*0]
    movq               xmm1, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pshufb             xmm0, xm6
    pshufb             xmm1, xm6
    mova               xmm2, xm5
    vpdpbusd           xmm2, xmm0, xmm3
    mova               xmm0, xm5
    vpdpbusd           xmm0, xmm1, xmm3
    packssdw           xmm0, xmm2, xmm0
    psraw              xmm0, 6
    packuswb           xmm0, xmm0
    movd       [dstq+dsq*0], xmm0
    pextrd     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w4_loop
    RET
.h_w8:
    movu                xm0, [srcq+ssq*0]
    vinserti32x4        ym0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    WRAP_YMM PUT_8TAP_H   0, 1, 2, 3
    vpmovuswb           xm0, ym0
    movq       [dstq+dsq*0], xm0
    movhps     [dstq+dsq*1], xm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w8
    RET
.h_w16:
    mova                 m6, [spel_h_perm16a]
    mova                 m7, [spel_h_perm16b]
    mova                 m8, [spel_h_perm16c]
.h_w16_loop:
    movu                ym0, [srcq+ssq*0]
    vinserti32x8         m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    PUT_8TAP_H            0, 1, 2, 3, 1
    vpmovuswb           ym0, m0
    mova         [dstq+dsq*0], xm0
    vextracti128 [dstq+dsq*1], ym0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w16_loop
    RET
.h_w32:
    movu                ym0, [srcq+ssq*0+8*0]
    vinserti32x8         m0, [srcq+ssq*1+8*0], 1
    movu                ym1, [srcq+ssq*0+8*1]
    vinserti32x8         m1, [srcq+ssq*1+8*1], 1
    lea                srcq, [srcq+ssq*2]
    PUT_8TAP_H            0, 2, 3, 4
    PUT_8TAP_H            1, 4, 3, 2
    packuswb             m0, m1
    mova          [dstq+dsq*0], ym0
    vextracti32x8 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .h_w32
    RET
.h_w64:
    movu                 m0, [srcq+8*0]
    movu                 m1, [srcq+8*1]
    add                srcq, ssq
    PUT_8TAP_H            0, 2, 3, 4
    PUT_8TAP_H            1, 4, 3, 2
    packuswb             m0, m1
    mova             [dstq], m0
    add                dstq, dsq
    dec                  hd
    jg .h_w64
    RET
.h_w128:
    movu                 m0, [srcq+8*0]
    movu                 m2, [srcq+8*1]
    movu                 m1, [srcq+8*8]
    movu                 m3, [srcq+8*9]
    add                srcq, ssq
    PUT_8TAP_H            0,  4, 11, 12
    PUT_8TAP_H            2, 12, 11,  4
    PUT_8TAP_H            1,  4, 11, 12
    PUT_8TAP_H            3, 12, 11,  4
    packuswb             m0, m2
    packuswb             m1, m3
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m1
    add                dstq, dsq
    dec                  hd
    jg .h_w128
    RET
.v:
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    tzcnt               r6d, wd
    movzx               r6d, word [r8+r6*2+table_offset(put, _8tap_v)]
    vpbroadcastd         m7, [pw_512]
    lea                 myq, [base+subpel_filters+myq*8]
    vpbroadcastw         m8, [myq+0]
    vpbroadcastw         m9, [myq+2]
    vpbroadcastw        m10, [myq+4]
    vpbroadcastw        m11, [myq+6]
    add                  r6, r8
    lea                ss3q, [ssq*3]
    sub                srcq, ss3q
    jmp                  r6
.v_w2:
    movd               xmm2, [srcq+ssq*0]
    pinsrw             xmm2, [srcq+ssq*1], 2
    pinsrw             xmm2, [srcq+ssq*2], 4
    add                srcq, ss3q
    pinsrw             xmm2, [srcq+ssq*0], 6  ; 0 1 2 3
    movd               xmm3, [srcq+ssq*1]
    vpbroadcastd       xmm1, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastd       xmm0, [srcq+ssq*0]
    vpblendd           xmm3, xmm3, xmm1, 0x02 ; 4 5
    vpblendd           xmm1, xmm1, xmm0, 0x02 ; 5 6
    palignr            xmm4, xmm3, xmm2, 4    ; 1 2 3 4
    punpcklbw          xmm3, xmm1             ; 45 56
    punpcklbw          xmm1, xmm2, xmm4       ; 01 12
    punpckhbw          xmm2, xmm4             ; 23 34
.v_w2_loop:
    pmaddubsw          xmm5, xmm1, xm8        ; a0 b0
    mova               xmm1, xmm2
    pmaddubsw          xmm2, xm9              ; a1 b1
    paddw              xmm5, xmm2
    mova               xmm2, xmm3
    pmaddubsw          xmm3, xm10             ; a2 b2
    paddw              xmm5, xmm3
    vpbroadcastd       xmm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vpblendd           xmm3, xmm0, xmm4, 0x02 ; 6 7
    vpbroadcastd       xmm0, [srcq+ssq*0]
    vpblendd           xmm4, xmm4, xmm0, 0x02 ; 7 8
    punpcklbw          xmm3, xmm4             ; 67 78
    pmaddubsw          xmm4, xmm3, xm11       ; a3 b3
    paddw              xmm5, xmm4
    pmulhrsw           xmm5, xm7
    packuswb           xmm5, xmm5
    pextrw     [dstq+dsq*0], xmm5, 0
    pextrw     [dstq+dsq*1], xmm5, 2
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w2_loop
    RET
.v_w4:
    movd               xmm2, [srcq+ssq*0]
    pinsrd             xmm2, [srcq+ssq*1], 1
    pinsrd             xmm2, [srcq+ssq*2], 2
    add                srcq, ss3q
    pinsrd             xmm2, [srcq+ssq*0], 3  ; 0 1 2 3
    movd               xmm3, [srcq+ssq*1]
    vpbroadcastd       xmm1, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastd       xmm0, [srcq+ssq*0]
    vpblendd           xmm3, xmm3, xmm1, 0x02 ; 4 5
    vpblendd           xmm1, xmm1, xmm0, 0x02 ; 5 6
    palignr            xmm4, xmm3, xmm2, 4    ; 1 2 3 4
    punpcklbw          xmm3, xmm1             ; 45 56
    punpcklbw          xmm1, xmm2, xmm4       ; 01 12
    punpckhbw          xmm2, xmm4             ; 23 34
.v_w4_loop:
    vpbroadcastd       xmm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw          xmm5, xmm1, xm8        ; a0 b0
    mova               xmm1, xmm2
    pmaddubsw          xmm2, xm9              ; a1 b1
    paddw              xmm5, xmm2
    mova               xmm2, xmm3
    pmaddubsw          xmm3, xm10             ; a2 b2
    paddw              xmm5, xmm3
    vpblendd           xmm3, xmm0, xmm4, 0x02 ; 6 7
    vpbroadcastd       xmm0, [srcq+ssq*0]
    vpblendd           xmm4, xmm4, xmm0, 0x02 ; 7 8
    punpcklbw          xmm3, xmm4             ; 67 78
    pmaddubsw          xmm4, xmm3, xm11       ; a3 b3
    paddw              xmm5, xmm4
    pmulhrsw           xmm5, xm7
    packuswb           xmm5, xmm5
    movd       [dstq+dsq*0], xmm5
    pextrd     [dstq+dsq*1], xmm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w4_loop
    RET
.v_w8:
    movq               xmm1, [srcq+ssq*0]
    vpbroadcastq       ymm0, [srcq+ssq*1]
    vpbroadcastq       ymm2, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq       ymm5, [srcq+ssq*0]
    vpbroadcastq       ymm3, [srcq+ssq*1]
    vpbroadcastq       ymm4, [srcq+ssq*2]
    add                srcq, ss3q
    vpblendd           ymm1, ymm0, 0x30
    vpblendd           ymm0, ymm2, 0x30
    punpcklbw          ymm1, ymm0 ; 01 12
    vpbroadcastq       ymm0, [srcq+ssq*0]
    vpblendd           ymm2, ymm5, 0x30
    vpblendd           ymm5, ymm3, 0x30
    punpcklbw          ymm2, ymm5 ; 23 34
    vpblendd           ymm3, ymm4, 0x30
    vpblendd           ymm4, ymm0, 0x30
    punpcklbw          ymm3, ymm4 ; 45 56
.v_w8_loop:
    vpbroadcastq       ymm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    pmaddubsw          ymm5, ymm1, ym8  ; a0 b0
    mova               ymm1, ymm2
    pmaddubsw          ymm2, ym9        ; a1 b1
    paddw              ymm5, ymm2
    mova               ymm2, ymm3
    pmaddubsw          ymm3, ym10       ; a2 b2
    paddw              ymm5, ymm3
    vpblendd           ymm3, ymm0, ymm4, 0x30
    vpbroadcastq       ymm0, [srcq+ssq*0]
    vpblendd           ymm4, ymm4, ymm0, 0x30
    punpcklbw          ymm3, ymm4       ; 67 78
    pmaddubsw          ymm4, ymm3, ym11 ; a3 b3
    paddw              ymm5, ymm4
    pmulhrsw           ymm5, ym7
    vextracti128       xmm4, ymm5, 1
    packuswb           xmm5, xmm4
    movq       [dstq+dsq*0], xmm5
    movhps     [dstq+dsq*1], xmm5
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w8_loop
    vzeroupper
    RET
.v_w16:
    mova                m12, [spel_v_perm16]
    vbroadcasti32x4      m1, [srcq+ssq*0]
    vbroadcasti32x4     ym4, [srcq+ssq*1]
    mov                 r6d, 0x0f
    vbroadcasti32x4      m2, [srcq+ssq*2]
    add                srcq, ss3q
    vbroadcasti32x4     ym5, [srcq+ssq*0]
    kmovb                k1, r6d
    vbroadcasti32x4      m3, [srcq+ssq*1]
    vbroadcasti32x4     ym6, [srcq+ssq*2]
    add                srcq, ss3q
    vbroadcasti32x4      m0, [srcq+ssq*0]
    vshufpd          m1{k1}, m4, m2, 0xcc
    vshufpd          m2{k1}, m5, m3, 0xcc
    vshufpd          m3{k1}, m6, m0, 0xcc
    vpermb               m1, m12, m1 ; 01 12
    vpermb               m2, m12, m2 ; 23 34
    vpermb               m3, m12, m3 ; 45 56
.v_w16_loop:
    pmaddubsw            m4, m1, m8  ; a0 b0
    mova                 m1, m2
    pmaddubsw            m5, m2, m9  ; a1 b1
    mova                 m2, m3
    pmaddubsw            m6, m3, m10 ; a2 b2
    mova                 m3, m0
    paddw                m4, m5
    vbroadcasti32x4     ym5, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vbroadcasti32x4      m0, [srcq+ssq*0]
    vshufpd          m3{k1}, m5, m0, 0xcc
    vpermb               m3, m12, m3 ; 67 78
    pmaddubsw            m5, m3, m11 ; a3 b3
    paddw                m4, m6
    paddw                m4, m5
    pmulhrsw             m4, m7
    vextracti32x8       ym5, m4, 1
    packuswb            ym4, ym5
    mova          [dstq+dsq*0], xm4
    vextracti32x4 [dstq+dsq*1], ym4, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w16_loop
    RET
.v_w32:
    mova                m12, [spel_v_perm32]
    pmovzxbq            m14, [pb_02461357]
    vpshrdw             m13, m12, m12, 8
    movu                ym0, [srcq+ssq*0]
    vinserti32x8         m0, [srcq+ssq*1], 1
    vpermb               m1, m12, m0 ; 01
    vinserti32x8         m0, [srcq+ssq*2], 0
    add                srcq, ss3q
    vpermb               m2, m13, m0 ; 12
    vinserti32x8         m0, [srcq+ssq*0], 1
    vpermb               m3, m12, m0 ; 23
    vinserti32x8         m0, [srcq+ssq*1], 0
    vpermb               m4, m13, m0 ; 34
    vinserti32x8         m0, [srcq+ssq*2], 1
    add                srcq, ss3q
    vpermb               m5, m12, m0 ; 45
    vinserti32x8         m0, [srcq+ssq*0], 0
    vpermb               m6, m13, m0 ; 56
.v_w32_loop:
    vinserti32x8         m0, [srcq+ssq*1], 1
    lea                srcq, [srcq+ssq*2]
    pmaddubsw           m15, m1, m8
    mova                 m1, m3
    pmaddubsw           m16, m2, m8
    mova                 m2, m4
    pmaddubsw           m17, m3, m9
    mova                 m3, m5
    pmaddubsw           m18, m4, m9
    mova                 m4, m6
    pmaddubsw           m19, m5, m10
    vpermb               m5, m12, m0 ; 67
    vinserti32x8         m0, [srcq+ssq*0], 0
    pmaddubsw           m20, m6, m10
    vpermb               m6, m13, m0 ; 78
    paddw               m15, m17
    pmaddubsw           m17, m5, m11
    paddw               m16, m18
    pmaddubsw           m18, m6, m11
    paddw               m15, m19
    paddw               m16, m20
    paddw               m15, m17
    paddw               m16, m18
    pmulhrsw            m15, m7
    pmulhrsw            m16, m7
    packuswb            m15, m16
    vpermq              m15, m14, m15
    mova          [dstq+dsq*0], ym15
    vextracti32x8 [dstq+dsq*1], m15, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_w32_loop
    vzeroupper
    RET
.v_w64:
.v_w128:
    lea                 r6d, [hq+wq*4-256]
    mov                  r4, srcq
    mov                  r7, dstq
.v_loop0:
    movu                 m2, [srcq+ssq*0]
    movu                 m4, [srcq+ssq*1]
    movu                 m6, [srcq+ssq*2]
    add                srcq, ss3q
    movu                m13, [srcq+ssq*0]
    movu                m15, [srcq+ssq*1]
    movu                m17, [srcq+ssq*2]
    add                srcq, ss3q
    movu                 m0, [srcq+ssq*0]
    punpcklbw            m1, m2, m4    ; 01l
    punpckhbw            m2, m4        ; 01h
    punpcklbw            m3, m4, m6    ; 12l
    punpckhbw            m4, m6        ; 12h
    punpcklbw            m5, m6, m13   ; 23l
    punpckhbw            m6, m13       ; 23h
    punpcklbw           m12, m13, m15  ; 34l
    punpckhbw           m13, m15       ; 34h
    punpcklbw           m14, m15, m17  ; 45l
    punpckhbw           m15, m17       ; 45h
    punpcklbw           m16, m17, m0   ; 56l
    punpckhbw           m17, m0        ; 56h
.v_loop:
    pmaddubsw           m18, m1, m8    ; a0l
    mova                 m1, m5
    pmaddubsw           m19, m2, m8    ; a0h
    mova                 m2, m6
    pmaddubsw           m20, m3, m8    ; b0l
    mova                 m3, m12
    pmaddubsw           m21, m4, m8    ; b0h
    mova                 m4, m13
    pmaddubsw            m5, m9        ; a1l
    pmaddubsw            m6, m9        ; a1h
    pmaddubsw           m12, m9        ; b1l
    pmaddubsw           m13, m9        ; b1h
    paddw               m18, m5
    mova                 m5, m14
    pmaddubsw           m14, m10       ; a2l
    paddw               m19, m6
    mova                 m6, m15
    pmaddubsw           m15, m10       ; a2h
    paddw               m20, m12
    mova                m12, m16
    pmaddubsw           m16, m10       ; b2l
    paddw               m21, m13
    mova                m13, m17
    pmaddubsw           m17, m10       ; b2h
    paddw               m18, m14
    paddw               m19, m15
    paddw               m20, m16
    paddw               m21, m17
    movu                m17, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    punpcklbw           m14, m0, m17  ; 67l
    punpckhbw           m15, m0, m17  ; 67h
    pmaddubsw           m16, m14, m11 ; a3l
    pmaddubsw            m0, m15, m11 ; a3h
    paddw               m18, m16
    paddw               m19, m0
    movu                 m0, [srcq+ssq*0]
    punpcklbw           m16, m17, m0  ; 78l
    punpckhbw           m17, m0       ; 78h
    pmulhrsw            m18, m7
    pmulhrsw            m19, m7
    packuswb            m18, m19
    mova       [dstq+dsq*0], m18
    pmaddubsw           m18, m16, m11 ; b3l
    pmaddubsw           m19, m17, m11 ; b3h
    paddw               m18, m20
    paddw               m19, m21
    pmulhrsw            m18, m7
    pmulhrsw            m19, m7
    packuswb            m18, m19
    mova       [dstq+dsq*1], m18
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .v_loop
    add                  r4, 64
    add                  r7, 64
    movzx                hd, r6b
    mov                srcq, r4
    mov                dstq, r7
    sub                 r6d, 256
    jg .v_loop0
    vzeroupper
    RET
.hv:
    cmp                  wd, 4
    jg .hv_w8
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m7, [base+subpel_filters+mxq*8+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastd         m8, [pd_2]
    vpbroadcastq        ym0, [base+subpel_filters+myq*8]
    lea                ss3q, [ssq*3]
    vpbroadcastd        ym9, [pd_32768]
    mov                  r6, srcq
    punpcklbw           ym0, ym8, ym0
    sub                  r6, ss3q
    psraw               ym0, 2 ; << 6
    mova               xm14, [spel_hv_end]
    pshufd             ym10, ym0, q0000
    pshufd             ym11, ym0, q1111
    pshufd             ym12, ym0, q2222
    pshufd             ym13, ym0, q3333
    cmp                  wd, 4
    je .hv_w4
    vbroadcasti128      ym6, [subpel_h_shuf4]
    movq               xmm2, [r6+ssq*0]
    movhps             xmm2, [r6+ssq*1]
    movq               xmm0, [r6+ssq*2]
    movhps             xmm0, [srcq+ssq*0]
    vpbroadcastq       ymm3, [srcq+ssq*1]
    vpbroadcastq       ymm4, [srcq+ssq*2]
    add                srcq, ss3q
    vpbroadcastq       ymm1, [srcq+ssq*0]
    vpblendd           ymm2, ymm3, 0x30
    vpblendd           ymm0, ymm1, 0x30 ; 2 3   6 _
    vpblendd           ymm2, ymm4, 0xc0 ; 0 1   4 5
    pshufb             ymm2, ym6
    pshufb             ymm0, ym6
    mova               ymm1, ym8
    vpdpbusd           ymm1, ymm2, ym7
    mova               ymm2, ym8
    vpdpbusd           ymm2, ymm0, ym7
    packssdw           ymm2, ymm1, ymm2
    psraw              ymm2, 2
    vextracti128       xmm3, ymm2, 1
    palignr            xmm4, xmm3, xmm2, 4
    punpcklwd          xmm1, xmm2, xmm4 ; 01 12
    punpckhwd          xmm2, xmm4       ; 23 34
    pshufd             xmm0, xmm3, q2121
    punpcklwd          xmm3, xmm0       ; 45 56
.hv_w2_loop:
    movq               xmm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    movhps             xmm4, [srcq+ssq*0]
    mova               xmm5, xm9
    vpdpwssd           xmm5, xmm1, xm10 ; a0 b0
    mova               xmm1, xmm2
    vpdpwssd           xmm5, xmm2, xm11 ; a1 b1
    pshufb             xmm4, xm6
    mova               xmm2, xmm3
    vpdpwssd           xmm5, xmm3, xm12 ; a2 b2
    mova               xmm3, xm8
    vpdpbusd           xmm3, xmm4, xm7
    packssdw           xmm4, xmm3, xmm3
    psraw              xmm4, 2
    palignr            xmm3, xmm4, xmm0, 12
    mova               xmm0, xmm4
    punpcklwd          xmm3, xmm4       ; 67 78
    vpdpwssd           xmm5, xmm3, xm13 ; a3 b3
    packuswb           xmm5, xmm5
    pshufb             xmm5, xm14
    pextrw     [dstq+dsq*0], xmm5, 0
    pextrw     [dstq+dsq*1], xmm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w2_loop
    vzeroupper
    RET
.hv_w4:
    movq               xmm1, [r6+ssq*0]
    vpbroadcastq        ym2, [r6+ssq*1]
    vinserti32x4        ym1, ymm1, [r6+ssq*2], 1
    vinserti32x4         m2, [srcq+ssq*0], 2
    vinserti32x4         m1, [srcq+ssq*1], 2
    vinserti32x4         m2, [srcq+ssq*2], 3 ; _ 1 3 5
    vbroadcasti32x4      m6, [subpel_h_shufA]
    add                srcq, ss3q
    vinserti32x4         m1, [srcq+ssq*0], 3 ; 0 2 4 6
    pshufb               m2, m6
    pshufb               m1, m6
    mova                 m0, m8
    vpdpbusd             m0, m2, m7
    mova                 m4, m8
    vpdpbusd             m4, m1, m7
    mova                ym1, [spel_hv_perm4a]
    mova                ym2, [spel_hv_perm4b]
    mova                ym3, [spel_hv_perm4c]
    packssdw             m0, m4
    psraw                m0, 2 ; _ 0   1 2   3 4   5 6
    mov                 r6d, 0x5555
    vpermb              ym1, ym1, ym0 ; 01 12
    vpermb               m2, m2, m0   ; 23 34
    vpermb               m3, m3, m0   ; 45 56
    kmovw                k1, r6d
    mova               ym15, [spel_hv_perm4d]
.hv_w4_loop:
    movq               xmm4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vinserti32x4        ym4, ymm4, [srcq+ssq*0], 1
    mova                ym5, ym9
    vpdpwssd            ym5, ym1, ym10 ; a0 b0
    mova                ym1, ym2
    pshufb              ym4, ym6
    mova                ym0, ym8
    vpdpbusd            ym0, ym4, ym7
    vpdpwssd            ym5, ym2, ym11 ; a1 b1
    mova                ym2, ym3
    vpdpwssd            ym5, ym3, ym12 ; a2 b2
    vpsraw          ym3{k1}, ym0, 2    ; 7 8
    vpermb              ym3, ym15, ym3 ; 67 78
    vpdpwssd            ym5, ym3, ym13 ; a3 b3
    packuswb            ym5, ym5
    vpermb              ym5, ym14, ym5
    movd       [dstq+dsq*0], xm5
    pextrd     [dstq+dsq*1], xm5, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w4_loop
    RET
.hv_w8:
    shr                 mxd, 16
    sub                srcq, 3
    vpbroadcastd        m10, [base+subpel_filters+mxq*8+0]
    vpbroadcastd        m11, [base+subpel_filters+mxq*8+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 6
    cmovs               myd, mxd
    vpbroadcastd         m8, [pd_2]
    vpbroadcastq         m0, [base+subpel_filters+myq*8]
    vpbroadcastd         m9, [pd_32768]
    punpcklbw            m0, m8, m0
    lea                ss3q, [ssq*3]
    psraw                m0, 2 ; << 6
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
    cmp                  wd, 8
    jne .hv_w16
    mov                  r6, srcq
    sub                  r6, ss3q
    movu               xmm1, [r6+ssq*0]
    vinserti128        ymm1, [r6+ssq*1], 1
    movu               xmm2, [srcq+ssq*1]
    vinserti32x4         m6, zmm1, [r6+ssq*2], 2
    vinserti128        ymm2, [srcq+ssq*2], 1
    vinserti32x4         m6, [srcq+ssq*0], 3 ; 0 1 2 3
    add                srcq, ss3q
    vbroadcasti32x4      m4, [subpel_h_shufA]
    vinserti32x4         m0, zmm2, [srcq+ssq*0], 2 ; 4 5 6 _
    vbroadcasti32x4      m7, [subpel_h_shufB]
    vbroadcasti32x4     m17, [subpel_h_shufC]
    pshufb               m1, m6, m4  ; 0 1 2 3   0123
    mova                 m2, m8
    vpdpbusd             m2, m1, m10
    pshufb               m5, m6, m7  ; 0 1 2 3   4567
    mova                 m1, m8
    vpdpbusd             m1, m5, m10
    pshufb               m4, m0, m4  ; 4 5 6 _   0123
    mova                 m3, m8
    vpdpbusd             m3, m4, m10
    pshufb               m7, m0, m7  ; 4 5 6 _   4567
    mova                 m4, m8
    vpdpbusd             m4, m7, m10
    pshufb               m6, m17
    vpdpbusd             m2, m5, m11
    vpdpbusd             m1, m6, m11
    pshufb               m6, m0, m17
    vpdpbusd             m3, m7, m11
    vpdpbusd             m4, m6, m11
    mova                 m5, [spel_hv_perm8a]
    mova                 m0, [spel_hv_perm8b]
    mov                  r6, 0x55555555ff00
    packssdw             m2, m1
    packssdw             m3, m4
    mova                m18, [spel_hv_perm8c]
    psraw                m2, 2 ; 0 1 2 3
    psraw                m3, 2 ; 4 5 6 _
    vpermb               m1, m5, m2 ; 01 12
    vbroadcasti32x8      m6, [subpel_h_shufA]
    kmovq                k1, r6
    vpermt2b             m2, m0, m3 ; 23 34
    vbroadcasti32x8      m7, [subpel_h_shufB]
    kshiftrq             k2, k1, 16
    mova               xm16, [spel_hv_end]
    vpermb               m3, m5, m3 ; 45 56
.hv_w8_loop:
    vbroadcasti32x4     ym4, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vbroadcasti32x4  m4{k1}, [srcq+ssq*0]
    mova                 m0, m9
    vpdpwssd             m0, m1, m12 ; a0 b0
    pshufb               m1, m4, m6  ; 7 8   0123 4567
    mova                 m5, m8
    vpdpbusd             m5, m1, m10
    pshufb               m4, m7      ; 7 8   4567 89ab
    vpdpwssd             m0, m2, m13 ; a1 b1
    mova                 m1, m2
    vpdpbusd             m5, m4, m11
    mova                 m2, m3
    vpdpwssd             m0, m3, m14 ; a2 b2
    psraw            m3{k2}, m5, 2   ; 75 86
    vpermb               m3, m18, m3 ; 67 78
    vpdpwssd             m0, m3, m15 ; a3 b3
    packuswb             m0, m0
    vpermb             zmm1, m16, m0
    movq       [dstq+dsq*0], xmm1
    movhps     [dstq+dsq*1], xmm1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w8_loop
    vzeroupper
    RET
.hv_w16:
    movu                 m7, [spel_hv_perm16a]
    sub                srcq, ss3q
    mova                m20, [spel_hv_perm16b]
    lea                 r6d, [wq*2-32]
    mova                m21, [spel_hv_perm16c]
    mov                  r4, srcq
    mov                  r7, dstq
    mova               ym16, [spel_hv_end16]
    lea                 r6d, [hq+r6*8]
.hv_w16_loop0:
    movu               ym17, [srcq+ssq*0]
    vinserti32x8        m17, [srcq+ssq*1], 1 ; 0 1
    movu               ym18, [srcq+ssq*2]
    add                srcq, ss3q
    vinserti32x8        m18, [srcq+ssq*0], 1 ; 2 3
    movu               ym19, [srcq+ssq*1]
    vinserti32x8        m19, [srcq+ssq*2], 1 ; 4 5
    add                srcq, ss3q
    vpermb               m2, m7, m17     ; 0 1   0123   89ab
    vpermb               m0, m20, m17    ; 0 1   4567   cdef
    vpermb               m4, m7, m18     ; 2 3   0123   89ab
    mova                 m1, m8
    vpdpbusd             m1, m2, m10
    vpermb               m5, m20, m18    ; 2 3   4567   cdef
    mova                 m2, m8
    vpdpbusd             m2, m0, m10
    vpermb              m17, m21, m17    ; 0 1   89ab   ghij
    mova                 m3, m8
    vpdpbusd             m3, m4, m10
    vpermb               m6, m7, m19     ; 4 5   0123   89ab
    mova                 m4, m8
    vpdpbusd             m4, m5, m10
    vpermb              m18, m21, m18    ; 2 3   89ab   ghij
    vpdpbusd             m1, m0, m11
    movu                ym0, [srcq+ssq*0] ; 6
    vpdpbusd             m2, m17, m11
    vpermb              m17, m20, m19    ; 4 5   4567   cdef
    vpdpbusd             m3, m5, m11
    mova                 m5, m8
    vpdpbusd             m5, m6, m10
    mova                 m6, m8
    vpdpbusd             m6, m17, m10
    vpdpbusd             m4, m18, m11
    mova                m18, [spel_hv_perm16d]
    vpermb              m18, m18, m0     ; 6     0145   2367   89cd   abef
    vpdpbusd             m5, m17, m11
    vpermb              m19, m21, m19    ; 4 5   89ab   ghij
    mova                m17, m8
    vpdpbusd            m17, m18, m10
    mova                m18, [spel_hv_perm16e]
    vpermb               m0, m18, m0     ; 6     4589   67ab   cdgh   efij
    packssdw             m1, m2          ; 01
    vpdpbusd             m6, m19, m11
    packssdw             m3, m4          ; 23
    vpdpbusd            m17, m0, m11
    psraw                m1, 2
    packssdw             m5, m6          ; 45
    psraw                m3, 2
    vpshrdd              m2, m1, m3, 16  ; 12
    psraw                m5, 2
    vpshrdd              m4, m3, m5, 16  ; 34
    psraw               m17, 2
    vpshrdd              m6, m5, m17, 16 ; 56
.hv_w16_loop:
    movu               ym18, [srcq+ssq*1]
    lea                srcq, [srcq+ssq*2]
    vinserti32x8        m18, [srcq+ssq*0], 1
    mova                 m0, m9
    vpdpwssd             m0, m1, m12     ; a0
    vpermb               m1, m7, m18     ; 7 8   0123   89ab
    mova                m17, m9
    vpdpwssd            m17, m2, m12     ; b0
    vpermb               m2, m20, m18    ; 7 8   4567   cdef
    mova                m19, m8
    vpdpbusd            m19, m1, m10
    vpermb              m18, m21, m18
    mova                 m1, m8
    vpdpbusd             m1, m2, m10
    vpdpwssd             m0, m3, m13    ; a1
    vpdpwssd            m17, m4, m13    ; b1
    vpdpbusd            m19, m2, m11
    mova                 m2, m4
    vpdpbusd             m1, m18, m11
    mova                 m4, m6
    vpdpwssd             m0, m5, m14    ; a2
    vpdpwssd            m17, m6, m14    ; b2
    packssdw            m19, m1
    mova                 m1, m3
    mova                 m3, m5
    psraw                m6, m19, 2     ; 7 8
    vpshrdd              m5, m4, m6, 16 ; 6 7
    vpdpwssd            m17, m6, m15    ; b3
    vpdpwssd             m0, m5, m15    ; a3
    packuswb             m0, m17
    vpermb             zmm1, m16, m0
    mova         [dstq+dsq*0], xmm1
    vextracti128 [dstq+dsq*1], ymm1, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .hv_w16_loop
    add                  r4, 16
    add                  r7, 16
    movzx                hd, r6b
    mov                srcq, r4
    mov                dstq, r7
    sub                 r6d, 1<<8
    jg .hv_w16_loop0
    vzeroupper
    RET

%macro PREP_8TAP_H 0
    vpermb              m10, m5, m0
    vpermb              m11, m5, m1
    vpermb              m12, m6, m0
    vpermb              m13, m6, m1
    vpermb              m14, m7, m0
    vpermb              m15, m7, m1
    mova                 m0, m4
    vpdpbusd             m0, m10, m8
    mova                 m2, m4
    vpdpbusd             m2, m12, m8
    mova                 m1, m4
    vpdpbusd             m1, m11, m8
    mova                 m3, m4
    vpdpbusd             m3, m13, m8
    vpdpbusd             m0, m12, m9
    vpdpbusd             m2, m14, m9
    vpdpbusd             m1, m13, m9
    vpdpbusd             m3, m15, m9
    packssdw             m0, m2
    packssdw             m1, m3
    psraw                m0, 2
    psraw                m1, 2
    mova        [tmpq+64*0], m0
    mova        [tmpq+64*1], m1
%endmacro

%if WIN64
DECLARE_REG_TMP 6, 4
%else
DECLARE_REG_TMP 6, 7
%endif

%define PREP_8TAP_FN FN prep_8tap,

PREP_8TAP_FN sharp,          SHARP,   SHARP
PREP_8TAP_FN sharp_smooth,   SHARP,   SMOOTH
PREP_8TAP_FN smooth_sharp,   SMOOTH,  SHARP
PREP_8TAP_FN smooth,         SMOOTH,  SMOOTH
PREP_8TAP_FN sharp_regular,  SHARP,   REGULAR
PREP_8TAP_FN regular_sharp,  REGULAR, SHARP
PREP_8TAP_FN smooth_regular, SMOOTH,  REGULAR
PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH
PREP_8TAP_FN regular,        REGULAR, REGULAR

cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3
    imul                mxd, mxm, 0x010101
    add                 mxd, t0d ; 8tap_h, mx, 4tap_h
    imul                myd, mym, 0x010101
    add                 myd, t1d ; 8tap_v, my, 4tap_v
    lea                  r7, [prep_avx512icl]
    movsxd               wq, wm
    movifnidn            hd, hm
    test                mxd, 0xf00
    jnz .h
    test                myd, 0xf00
    jnz .v
    tzcnt                wd, wd
    movzx                wd, word [r7+wq*2+table_offset(prep,)]
    add                  wq, r7
    lea                  r6, [strideq*3]
%if WIN64
    pop                  r7
%endif
    jmp                  wq
.h:
    test                myd, 0xf00
    jnz .hv
    vpbroadcastd         m4, [pd_2]
    WIN64_SPILL_XMM      10
    cmp                  wd, 4
    je .h_w4
    tzcnt                wd, wd
    shr                 mxd, 16
    sub                srcq, 3
    movzx                wd, word [r7+wq*2+table_offset(prep, _8tap_h)]
    vpbroadcastd         m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
    vpbroadcastd         m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
    add                  wq, r7
    jmp                  wq
.h_w4:
    movzx               mxd, mxb
    vbroadcasti128      ym5, [subpel_h_shufA]
    mov                 r3d, 0x4
    dec                srcq
    vpbroadcastd        ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
    kmovb                k1, r3d
    lea            stride3q, [strideq*3]
.h_w4_loop:
    movq                xm2, [srcq+strideq*0]
    movq                xm3, [srcq+strideq*1]
    vpbroadcastq    ym2{k1}, [srcq+strideq*2]
    vpbroadcastq    ym3{k1}, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    pshufb              ym2, ym5
    pshufb              ym3, ym5
    mova                ym0, ym4
    vpdpbusd            ym0, ym2, ym6
    mova                ym1, ym4
    vpdpbusd            ym1, ym3, ym6
    packssdw            ym0, ym1
    psraw               ym0, 2
    mova             [tmpq], ym0
    add                tmpq, 32
    sub                  hd, 4
    jg .h_w4_loop
    RET
.h_w8:
    vbroadcasti128       m5, [subpel_h_shufA]
    vbroadcasti128       m6, [subpel_h_shufB]
    vbroadcasti128       m7, [subpel_h_shufC]
    lea            stride3q, [strideq*3]
.h_w8_loop:
    movu               xmm3, [srcq+strideq*0]
    vinserti128         ym3, ymm3, [srcq+strideq*1], 1
    vinserti128          m3, [srcq+strideq*2], 2
    vinserti128          m3, [srcq+stride3q ], 3
    lea                srcq, [srcq+strideq*4]
    pshufb               m1, m3, m5
    pshufb               m2, m3, m6
    mova                 m0, m4
    vpdpbusd             m0, m1, m8
    mova                 m1, m4
    vpdpbusd             m1, m2, m8
    pshufb               m3, m7
    vpdpbusd             m0, m2, m9
    vpdpbusd             m1, m3, m9
    packssdw             m0, m1
    psraw                m0, 2
    mova             [tmpq], m0
    add                tmpq, 64
    sub                  hd, 4
    jg .h_w8_loop
    RET
.h_w16:
    mova                 m5, [spel_h_perm16a]
    mova                 m6, [spel_h_perm16b]
    mova                 m7, [spel_h_perm16c]
    lea            stride3q, [strideq*3]
.h_w16_loop:
    movu                ym0, [srcq+strideq*0]
    movu                ym1, [srcq+strideq*2]
    vinserti32x8         m0, [srcq+strideq*1], 1
    vinserti32x8         m1, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    PREP_8TAP_H
    add                tmpq, 64*2
    sub                  hd, 4
    jg .h_w16_loop
    RET
.h_w32:
    mova                 m5, [spel_h_perm32a]
    mova                 m6, [spel_h_perm32b]
    mova                 m7, [spel_h_perm32c]
.h_w32_loop:
    movu                 m0, [srcq+strideq*0]
    movu                 m1, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    PREP_8TAP_H
    add                tmpq, 64*2
    sub                  hd, 2
    jg .h_w32_loop
    RET
.h_w64:
    xor                 r6d, r6d
    jmp .h_start
.h_w128:
    mov                  r6, -64*1
.h_start:
    mova                 m5, [spel_h_perm32a]
    mova                 m6, [spel_h_perm32b]
    mova                 m7, [spel_h_perm32c]
    sub                srcq, r6
    mov                  r5, r6
.h_loop:
    movu                 m0, [srcq+r6+32*0]
    movu                 m1, [srcq+r6+32*1]
    PREP_8TAP_H
    add                tmpq, 64*2
    add                  r6, 64
    jle .h_loop
    add                srcq, strideq
    mov                  r6, r5
    dec                  hd
    jg .h_loop
    RET
.v:
    movzx               mxd, myb ; Select 4-tap/8-tap filter multipliers.
    shr                 myd, 16  ; Note that the code is 8-tap only, having
    tzcnt                wd, wd
    cmp                  hd, 4   ; a separate 4-tap code path for (4|8|16)x4
    cmove               myd, mxd ; had a negligible effect on performance.
    ; TODO: Would a 6-tap code path be worth it?
    lea                 myq, [r7+myq*8+subpel_filters-prep_avx512icl]
    movzx                wd, word [r7+wq*2+table_offset(prep, _8tap_v)]
    add                  wq, r7
    lea            stride3q, [strideq*3]
    sub                srcq, stride3q
    vpbroadcastd         m7, [pw_8192]
    vpbroadcastw         m8, [myq+0]
    vpbroadcastw         m9, [myq+2]
    vpbroadcastw        m10, [myq+4]
    vpbroadcastw        m11, [myq+6]
    jmp                  wq
.v_w4:
    movd               xmm0, [srcq+strideq*0]
    vpbroadcastd       ymm1, [srcq+strideq*2]
    vpbroadcastd       xmm2, [srcq+strideq*1]
    vpbroadcastd       ymm3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vpblendd           ymm1, ymm0, 0x01       ; 0 2 2 _   2 _ _ _
    vpblendd           ymm3, ymm2, 0x03       ; 1 1 3 3   3 3 _ _
    vpbroadcastd       ymm0, [srcq+strideq*0]
    vpbroadcastd       ymm2, [srcq+strideq*1]
    vpblendd           ymm1, ymm0, 0x68       ; 0 2 2 4   2 4 4 _
    vpbroadcastd       ymm0, [srcq+strideq*2]
    vbroadcasti128     ymm5, [deint_shuf4]
    vpblendd           ymm3, ymm2, 0xc0       ; 1 1 3 3   3 3 5 5
    vpblendd           ymm2, ymm3, ymm1, 0x55 ; 0 1 2 3   2 3 4 5
    vpblendd           ymm3, ymm1, 0xaa       ; 1 2 3 4   3 4 5 _
    punpcklbw          ymm1, ymm2, ymm3       ; 01  12    23  34
    vpblendd           ymm3, ymm0, 0x80       ; 1 2 3 4   3 4 5 6
    punpckhbw          ymm2, ymm3             ; 23  34    45  56
.v_w4_loop:
    pinsrd             xmm0, [srcq+stride3q ], 1
    lea                srcq, [srcq+strideq*4]
    vpbroadcastd       ymm3, [srcq+strideq*0]
    vpbroadcastd       ymm4, [srcq+strideq*1]
    vpblendd           ymm3, ymm4, 0x20       ; _ _ 8 _   8 9 _ _
    vpblendd           ymm3, ymm0, 0x03       ; 6 7 8 _   8 9 _ _
    vpbroadcastd       ymm0, [srcq+strideq*2]
    vpblendd           ymm3, ymm0, 0x40       ; 6 7 8 _   8 9 a _
    pshufb             ymm3, ymm5             ; 67  78    89  9a
    pmaddubsw          ymm4, ymm1, ym8
    vperm2i128         ymm1, ymm2, ymm3, 0x21 ; 45  56    67  78
    pmaddubsw          ymm2, ym9
    paddw              ymm4, ymm2
    mova               ymm2, ymm3
    pmaddubsw          ymm3, ym11
    paddw              ymm3, ymm4
    pmaddubsw          ymm4, ymm1, ym10
    paddw              ymm3, ymm4
    pmulhrsw           ymm3, ym7
    mova             [tmpq], ymm3
    add                tmpq, 32
    sub                  hd, 4
    jg .v_w4_loop
    vzeroupper
    RET
.v_w8:
    mov                 r3d, 0xf044
    kmovw                k1, r3d
    kshiftrw             k2, k1, 8
    movq                xm0, [srcq+strideq*0]
    vpbroadcastq        ym1, [srcq+strideq*1]
    vpbroadcastq         m2, [srcq+strideq*2]
    vpbroadcastq         m3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vpbroadcastq         m4, [srcq+strideq*0]
    vpbroadcastq         m5, [srcq+strideq*1]
    vpbroadcastq         m6, [srcq+strideq*2]
    vmovdqa64       ym0{k1}, ym1
    vmovdqa64       ym1{k1}, ym2
    vmovdqa64        m2{k1}, m3
    vmovdqa64        m3{k1}, m4
    vmovdqa64        m4{k1}, m5
    vmovdqa64        m5{k1}, m6
    punpcklbw           ym0, ym1 ; 01 12 __ __
    punpcklbw            m2, m3  ; 23 34 23 34
    punpcklbw            m4, m5  ; 45 56 45 56
    vmovdqa64        m0{k2}, m2  ; 01 12 23 34
    vmovdqa64        m2{k2}, m4  ; 23 34 45 56
.v_w8_loop:
    vpbroadcastq         m1, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vpbroadcastq         m3, [srcq+strideq*0]
    vpbroadcastq         m5, [srcq+strideq*1]
    pmaddubsw           m14, m0, m8
    pmaddubsw           m15, m2, m9
    vpblendmq        m0{k1}, m6, m1
    vpblendmq        m2{k1}, m1, m3
    vpbroadcastq         m6, [srcq+strideq*2]
    paddw               m14, m15
    punpcklbw            m2, m0, m2   ; 67 78 67 78
    vpblendmq       m12{k1}, m3, m5
    vpblendmq       m13{k1}, m5, m6
    vpblendmq        m0{k2}, m4, m2   ; 45 56 67 78
    punpcklbw            m4, m12, m13 ; 89 9a 89 9a
    vmovdqa64        m2{k2}, m4       ; 67 78 89 9a
    pmaddubsw           m12, m0, m10
    pmaddubsw           m13, m2, m11
    paddw               m14, m12
    paddw               m14, m13
    pmulhrsw            m14, m7
    mova             [tmpq], m14
    add                tmpq, 64
    sub                  hd, 4
    jg .v_w8_loop
    RET
.v_w16:
    mov                 r3d, 0xf0
    kmovb                k1, r3d
    vbroadcasti128       m0, [srcq+strideq*0]
    vbroadcasti128       m1, [srcq+strideq*1]
    vbroadcasti128       m2, [srcq+strideq*2]
    vbroadcasti128       m3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vbroadcasti128       m4, [srcq+strideq*0]
    vbroadcasti128       m5, [srcq+strideq*1]
    vbroadcasti128       m6, [srcq+strideq*2]
    vmovdqa64        m0{k1}, m1
    vmovdqa64        m1{k1}, m2
    vmovdqa64        m2{k1}, m3
    vmovdqa64        m3{k1}, m4
    vmovdqa64        m4{k1}, m5
    vmovdqa64        m5{k1}, m6
    shufpd               m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b
    shufpd               m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b
    shufpd               m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_--
    shufpd               m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_--
    punpckhbw            m2, m0, m1   ;  23a   23b   34a   34b
    punpcklbw            m0, m1       ;  01a   01b   12a   12b
    punpcklbw            m4, m5       ;  45a   45b   56a   56b
.v_w16_loop:
    vbroadcasti128       m3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vbroadcasti128       m5, [srcq+strideq*0]
    vpblendmq        m1{k1}, m6, m3
    vmovdqa64        m3{k1}, m5
    pmaddubsw           m12, m0, m8
    pmaddubsw           m13, m2, m8
    pmaddubsw           m14, m2, m9
    pmaddubsw           m15, m4, m9
    pmaddubsw            m0, m4, m10
    vbroadcasti128       m2, [srcq+strideq*1]
    vbroadcasti128       m6, [srcq+strideq*2]
    paddw               m12, m14
    paddw               m13, m15
    paddw               m12, m0
    vmovdqa64        m5{k1}, m2
    vmovdqa64        m2{k1}, m6
    mova                 m0, m4
    shufpd               m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b
    shufpd               m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab
    punpcklbw            m2, m1, m3   ;  67a   67b   78a   78b
    punpckhbw            m4, m1, m3   ;  89a   89b   9Aa   9Ab
    pmaddubsw           m14, m2, m10
    pmaddubsw           m15, m2, m11
    paddw               m13, m14
    paddw               m12, m15
    pmaddubsw           m14, m4, m11
    paddw               m13, m14
    pmulhrsw            m12, m7
    pmulhrsw            m13, m7
    mova          [tmpq+ 0], m12
    mova          [tmpq+64], m13
    add                tmpq, 64*2
    sub                  hd, 4
    jg .v_w16_loop
    RET
.v_w32:
    mova                m18, [bilin_v_perm64]
    movu                ym0, [srcq+strideq*0]
    movu                ym1, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    movu                ym2, [srcq+strideq*0]
    movu                ym3, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    movu                ym4, [srcq+strideq*0]
    movu                ym5, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    movu                ym6, [srcq+strideq*0]
    vpermq               m0, m18, m0
    vpermq               m1, m18, m1
    vpermq               m2, m18, m2
    vpermq               m3, m18, m3
    vpermq               m4, m18, m4
    vpermq               m5, m18, m5
    vpermq               m6, m18, m6
    punpcklbw            m0, m1
    punpcklbw            m1, m2
    punpcklbw            m2, m3
    punpcklbw            m3, m4
    punpcklbw            m4, m5
    punpcklbw            m5, m6
.v_w32_loop:
    movu               ym12, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    movu               ym13, [srcq+strideq*0]
    pmaddubsw           m14, m0, m8
    pmaddubsw           m16, m2, m9
    pmaddubsw           m15, m1, m8
    pmaddubsw           m17, m3, m9
    mova                 m0, m2
    mova                 m1, m3
    vpermq              m12, m18, m12
    vpermq              m13, m18, m13
    paddw               m14, m16
    paddw               m15, m17
    pmaddubsw           m16, m4, m10
    pmaddubsw           m17, m5, m10
    punpcklbw            m6, m12
    punpcklbw           m12, m13
    mova                 m2, m4
    mova                 m3, m5
    paddw               m14, m16
    paddw               m15, m17
    pmaddubsw           m16, m6, m11
    pmaddubsw           m17, m12, m11
    mova                 m4, m6
    mova                 m5, m12
    paddw               m14, m16
    paddw               m15, m17
    pmulhrsw            m14, m7
    pmulhrsw            m15, m7
    mova                 m6, m13
    mova          [tmpq+ 0], m14
    mova          [tmpq+64], m15
    add                tmpq, 64*2
    sub                  hd, 2
    jg .v_w32_loop
    vzeroupper
    RET
.v_w64:
    mov                  wd, 64
    jmp .v_start
.v_w128:
    mov                  wd, 128
.v_start:
    WIN64_SPILL_XMM      27
    mova                m26, [bilin_v_perm64]
    lea                 r6d, [hq+wq*2]
    mov                  r5, srcq
    mov                  r7, tmpq
.v_loop0:
    vpermq               m0, m26, [srcq+strideq*0]
    vpermq               m1, m26, [srcq+strideq*1]
    lea                srcq,      [srcq+strideq*2]
    vpermq               m2, m26, [srcq+strideq*0]
    vpermq               m3, m26, [srcq+strideq*1]
    lea                srcq,      [srcq+strideq*2]
    vpermq               m4, m26, [srcq+strideq*0]
    vpermq               m5, m26, [srcq+strideq*1]
    lea                srcq,      [srcq+strideq*2]
    vpermq               m6, m26, [srcq+strideq*0]
    punpckhbw           m12, m0, m1
    punpcklbw            m0, m1
    punpckhbw           m13, m1, m2
    punpcklbw            m1, m2
    punpckhbw           m14, m2, m3
    punpcklbw            m2, m3
    punpckhbw           m15, m3, m4
    punpcklbw            m3, m4
    punpckhbw           m16, m4, m5
    punpcklbw            m4, m5
    punpckhbw           m17, m5, m6
    punpcklbw            m5, m6
.v_loop:
    vpermq              m18, m26, [srcq+strideq*1]
    lea                srcq,      [srcq+strideq*2]
    vpermq              m19, m26, [srcq+strideq*0]
    pmaddubsw           m20, m0, m8
    pmaddubsw           m21, m12, m8
    pmaddubsw           m22, m1, m8
    pmaddubsw           m23, m13, m8
    mova                 m0, m2
    mova                m12, m14
    mova                 m1, m3
    mova                m13, m15
    pmaddubsw            m2, m9
    pmaddubsw           m14, m9
    pmaddubsw            m3, m9
    pmaddubsw           m15, m9
    punpckhbw           m24, m6, m18
    punpcklbw            m6, m18
    paddw               m20, m2
    paddw               m21, m14
    paddw               m22, m3
    paddw               m23, m15
    mova                 m2, m4
    mova                m14, m16
    mova                 m3, m5
    mova                m15, m17
    pmaddubsw            m4, m10
    pmaddubsw           m16, m10
    pmaddubsw            m5, m10
    pmaddubsw           m17, m10
    punpckhbw           m25, m18, m19
    punpcklbw           m18, m19
    paddw               m20, m4
    paddw               m21, m16
    paddw               m22, m5
    paddw               m23, m17
    mova                 m4, m6
    mova                m16, m24
    mova                 m5, m18
    mova                m17, m25
    pmaddubsw            m6, m11
    pmaddubsw           m24, m11
    pmaddubsw           m18, m11
    pmaddubsw           m25, m11
    paddw               m20, m6
    paddw               m21, m24
    paddw               m22, m18
    paddw               m23, m25
    pmulhrsw            m20, m7
    pmulhrsw            m21, m7
    pmulhrsw            m22, m7
    pmulhrsw            m23, m7
    mova                 m6, m19
    mova     [tmpq+wq*0+ 0], m20
    mova     [tmpq+wq*0+64], m21
    mova     [tmpq+wq*2+ 0], m22
    mova     [tmpq+wq*2+64], m23
    lea                tmpq, [tmpq+wq*4]
    sub                  hd, 2
    jg .v_loop
    add                  r5, 64
    add                  r7, 128
    movzx                hd, r6b
    mov                srcq, r5
    mov                tmpq, r7
    sub                 r6d, 1<<8
    jg .v_loop0
    RET
.hv:
    %assign stack_offset stack_offset - stack_size_padded
    %assign stack_size_padded 0
    WIN64_SPILL_XMM      16
    cmp                  wd, 4
    je .hv_w4
    shr                 mxd, 16
    sub                srcq, 3
    vpbroadcastd        m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0]
    vpbroadcastd        m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    tzcnt                wd, wd
    vpbroadcastd         m8, [pd_2]
    movzx                wd, word [r7+wq*2+table_offset(prep, _8tap_hv)]
    vpbroadcastd         m9, [pd_32]
    add                  wq, r7
    vpbroadcastq         m0, [r7+myq*8+subpel_filters-prep_avx512icl]
    lea            stride3q, [strideq*3]
    sub                srcq, stride3q
    punpcklbw            m0, m0
    psraw                m0, 8 ; sign-extend
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
    jmp                  wq
.hv_w4:
    movzx               mxd, mxb
    dec                srcq
    vpbroadcastd         m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2]
    movzx               mxd, myb
    shr                 myd, 16
    cmp                  hd, 4
    cmove               myd, mxd
    vpbroadcastq         m0, [r7+myq*8+subpel_filters-prep_avx512icl]
    lea            stride3q, [strideq*3]
    sub                srcq, stride3q
    mov                 r3d, 0x04
    kmovb                k1, r3d
    kshiftlb             k2, k1, 2
    kshiftlb             k3, k1, 4
    vpbroadcastd        m10, [pd_2]
    vbroadcasti128      m16, [subpel_h_shufA]
    punpcklbw            m0, m0
    psraw                m0, 8 ; sign-extend
    vpbroadcastd        m11, [pd_32]
    pshufd              m12, m0, q0000
    pshufd              m13, m0, q1111
    pshufd              m14, m0, q2222
    pshufd              m15, m0, q3333
    movq                xm3, [srcq+strideq*0]
    vpbroadcastq        ym2, [srcq+strideq*1]
    vpbroadcastq    ym3{k1}, [srcq+strideq*2]
    vpbroadcastq     m2{k2}, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vpbroadcastq     m3{k2}, [srcq+strideq*0]
    vpbroadcastq     m2{k3}, [srcq+strideq*1]
    vpbroadcastq     m3{k3}, [srcq+strideq*2]
    mova                m17, [spel_hv_perm4a]
    movu                m18, [spel_hv_perm4b]
    mova                 m0, m10
    mova                 m1, m10
    pshufb               m2, m16
    pshufb               m3, m16
    vpdpbusd             m0, m2, m8
    vpdpbusd             m1, m3, m8
    packssdw             m0, m1        ; _ 0  1 2  3 4  5 6
    psraw                m0, 2
    vpermb               m1, m17, m0   ; 01 12 23 34
    vpermb               m2, m18, m0   ; 23 34 45 56
.hv_w4_loop:
    movq                xm3, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    movq                xm4, [srcq+strideq*0]
    vpbroadcastq    ym3{k1}, [srcq+strideq*1]
    vpbroadcastq    ym4{k1}, [srcq+strideq*2]
    mova                ym5, ym10
    mova                ym6, ym10
    pshufb              ym3, ym16
    pshufb              ym4, ym16
    vpdpbusd            ym5, ym3, ym8
    vpdpbusd            ym6, ym4, ym8
    mova                 m7, m11
    packssdw            ym5, ym6       ; 7 8  9 a  _ _  _ _
    psraw               ym5, 2
    valignq              m0, m5, m0, 4 ; _ 4  5 6  7 8  9 a
    vpdpwssd             m7, m1, m12
    vpdpwssd             m7, m2, m13
    vpermb               m1, m17, m0   ; 45 56 67 78
    vpermb               m2, m18, m0   ; 67 78 89 9a
    vpdpwssd             m7, m1, m14
    vpdpwssd             m7, m2, m15
    psrad                m7, 6
    vpmovdw          [tmpq], m7
    add                tmpq, 32
    sub                  hd, 4
    jg .hv_w4_loop
    vzeroupper
    RET
.hv_w8:
    WIN64_SPILL_XMM      24
    vbroadcasti128      m16, [subpel_h_shufA]
    vbroadcasti128      m17, [subpel_h_shufB]
    vbroadcasti128      m18, [subpel_h_shufC]
    vinserti128         ym0, [srcq+strideq*0], 1
    vinserti128          m0, [srcq+strideq*1], 2
    vinserti128          m0, [srcq+strideq*2], 3
    movu                xm1, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vinserti128         ym1, [srcq+strideq*0], 1
    vinserti128          m1, [srcq+strideq*1], 2
    vinserti128          m1, [srcq+strideq*2], 3
    mova                 m2, m8
    mova                 m4, m8
    mova                 m3, m8
    mova                 m5, m8
    pshufb              m20, m0, m16
    pshufb              m21, m0, m17
    pshufb              m22, m0, m18
    pshufb              m23, m1, m16
    pshufb               m6, m1, m17
    pshufb               m7, m1, m18
    vpdpbusd             m2, m20, m10
    vpdpbusd             m4, m21, m10
    vpdpbusd             m2, m21, m11
    vpdpbusd             m4, m22, m11
    vpdpbusd             m3, m23, m10
    vpdpbusd             m5,  m6, m10
    vpdpbusd             m3,  m6, m11
    vpdpbusd             m5,  m7, m11
    packssdw             m2, m4
    packssdw             m3, m5
    psraw                m2, 2          ; _ 0 1 2
    psraw                m3, 2          ; 3 4 5 6
    valignq              m0, m3, m2, 2  ; 0 1 2 3
    valignq              m1, m3, m2, 4  ; 1 2 3 4
    valignq              m2, m3, m2, 6  ; 2 3 4 5
    punpcklwd            m4, m0, m1 ; 01a 12a 23a 34a
    punpckhwd            m5, m0, m1 ; 01b 12b 23b 34b
    punpcklwd            m6, m2, m3 ; 23a 34a 45a 56a
    punpckhwd            m7, m2, m3 ; 23b 34b 45b 56b
.hv_w8_loop:
    movu               xm19, [srcq+stride3q ]
    lea                srcq, [srcq+strideq*4]
    vinserti128        ym19, [srcq+strideq*0], 1
    vinserti128         m19, [srcq+strideq*1], 2
    vinserti128         m19, [srcq+strideq*2], 3
    mova                m20, m9
    mova                m21, m9
    mova                m22, m8
    mova                m23, m8
    vpdpwssd            m20, m4, m12
    vpdpwssd            m21, m5, m12
    vpdpwssd            m20, m6, m13
    vpdpwssd            m21, m7, m13
    pshufb               m0, m19, m16
    pshufb               m1, m19, m17
    pshufb               m2, m19, m18
    vpdpbusd            m22, m0, m10
    vpdpbusd            m23, m1, m10
    vpdpbusd            m22, m1, m11
    vpdpbusd            m23, m2, m11
    packssdw            m22, m23
    psraw               m22, 2          ; 7 8 9 A
    valignq              m0, m22, m3, 2 ; 4 5 6 7
    valignq              m1, m22, m3, 4 ; 5 6 7 8
    valignq              m2, m22, m3, 6 ; 6 7 8 9
    mova                 m3, m22
    punpcklwd            m4, m0, m1 ; 45a 56a 67a 78a
    punpckhwd            m5, m0, m1 ; 45b 56b 67b 78b
    punpcklwd            m6, m2, m3 ; 67a 78a 89a 9Aa
    punpckhwd            m7, m2, m3 ; 67b 78b 89b 9Ab
    vpdpwssd            m20, m4, m14
    vpdpwssd            m21, m5, m14
    vpdpwssd            m20, m6, m15
    vpdpwssd            m21, m7, m15
    psrad               m20, 6
    psrad               m21, 6
    packssdw            m20, m21
    mova             [tmpq], m20
    add                tmpq, 64
    sub                  hd, 4
    jg .hv_w8_loop
    RET
.hv_w16:
    mov                  wd, 16*2
    jmp .hv_start
.hv_w32:
    mov                  wd, 32*2
    jmp .hv_start
.hv_w64:
    mov                  wd, 64*2
    jmp .hv_start
.hv_w128:
    mov                  wd, 128*2
.hv_start:
    WIN64_SPILL_XMM      31
    mova                m16, [spel_h_perm16a]
    mova                m17, [spel_h_perm16b]
    mova                m18, [spel_h_perm16c]
    lea                 r6d, [hq+wq*8-256]
    mov                  r5, srcq
    mov                  r7, tmpq
.hv_loop0:
    movu                ym0, [srcq+strideq*0]
    vinserti32x8         m0, [srcq+strideq*1], 1
    lea                srcq, [srcq+strideq*2]
    movu                ym1, [srcq+strideq*0]
    vinserti32x8         m1, [srcq+strideq*1], 1
    lea                srcq, [srcq+strideq*2]
    movu                ym2, [srcq+strideq*0]
    vinserti32x8         m2, [srcq+strideq*1], 1
    lea                srcq, [srcq+strideq*2]
    movu                ym3, [srcq+strideq*0]
    mova                 m4, m8
    mova                 m5, m8
    mova                 m6, m8
    mova                 m7, m8
    vpermb              m19, m16, m0
    vpermb              m20, m17, m0
    vpermb              m21, m18, m0
    vpermb              m22, m16, m1
    vpermb              m23, m17, m1
    vpermb              m24, m18, m1
    vpermb              m25, m16, m2
    vpermb              m26, m17, m2
    vpermb              m27, m18, m2
    vpermb             ym28, ym16, ym3
    vpermb             ym29, ym17, ym3
    vpermb             ym30, ym18, ym3
    mova                 m0, m8
    mova                 m1, m8
    mova                ym2, ym8
    mova                ym3, ym8
    vpdpbusd             m4, m19, m10
    vpdpbusd             m5, m20, m10
    vpdpbusd             m6, m22, m10
    vpdpbusd             m7, m23, m10
    vpdpbusd             m0, m25, m10
    vpdpbusd             m1, m26, m10
    vpdpbusd            ym2, ym28, ym10
    vpdpbusd            ym3, ym29, ym10
    vpdpbusd             m4, m20, m11
    vpdpbusd             m5, m21, m11
    vpdpbusd             m6, m23, m11
    vpdpbusd             m7, m24, m11
    vpdpbusd             m0, m26, m11
    vpdpbusd             m1, m27, m11
    vpdpbusd            ym2, ym29, ym11
    vpdpbusd            ym3, ym30, ym11
    packssdw             m4, m5
    packssdw             m6, m7
    packssdw             m0, m1
    packssdw            ym2, ym3
    psraw                m4, 2             ; 0a 0b 1a 1b
    psraw                m6, 2             ; 2a 2b 3a 3b
    psraw                m0, 2             ; 4a 4b 5a 5b
    psraw               ym2, 2             ; 6a 6b __ __
    vshufi32x4           m5, m4, m6, q1032 ; 1a 1b 2a 2b
    vshufi32x4           m7, m6, m0, q1032 ; 3a 3b 4a 4b
    vshufi32x4           m1, m0, m2, q1032 ; 5a 5b 6a 6b
    punpcklwd            m2, m4, m5 ; 01a 01c 12a 12c
    punpckhwd            m3, m4, m5 ; 01b 01d 12b 12d
    punpcklwd            m4, m6, m7 ; 23a 23c 34a 34c
    punpckhwd            m5, m6, m7 ; 23b 23d 34b 34d
    punpcklwd            m6, m0, m1 ; 45a 45c 56a 56c
    punpckhwd            m7, m0, m1 ; 45b 45d 56b 56d
.hv_loop:
    movu               ym19, [srcq+strideq*1]
    lea                srcq, [srcq+strideq*2]
    vinserti32x8        m19, [srcq+strideq*0], 1
    mova                m20, m9
    mova                m21, m9
    mova                m22, m8
    mova                m23, m8
    vpdpwssd            m20, m2, m12
    vpdpwssd            m21, m3, m12
    vpdpwssd            m20, m4, m13
    vpdpwssd            m21, m5, m13
    vpermb              m24, m16, m19
    vpermb              m25, m17, m19
    vpermb              m26, m18, m19
    vpdpbusd            m22, m24, m10
    vpdpbusd            m23, m25, m10
    vpdpbusd            m22, m25, m11
    vpdpbusd            m23, m26, m11
    packssdw            m22, m23
    psraw               m22, 2              ; 7a 7b 8a 8b
    vshufi32x4           m0, m1, m22, q1032 ; 6a 6b 7a 7b
    mova                 m2, m4
    mova                 m3, m5
    mova                 m1, m22
    mova                 m4, m6
    mova                 m5, m7
    punpcklwd            m6, m0, m1 ; 67a 67c 78a 78c
    punpckhwd            m7, m0, m1 ; 67b 67d 78b 78d
    vpdpwssd            m20, m4, m14
    vpdpwssd            m21, m5, m14
    vpdpwssd            m20, m6, m15
    vpdpwssd            m21, m7, m15
    psrad               m20, 6
    psrad               m21, 6
    packssdw            m20, m21
    mova          [tmpq+wq*0], ym20
    vextracti32x8 [tmpq+wq*1], m20, 1
    lea                tmpq, [tmpq+wq*2]
    sub                  hd, 2
    jg .hv_loop
    add                  r5, 16
    add                  r7, 32
    movzx                hd, r6b
    mov                srcq, r5
    mov                tmpq, r7
    sub                 r6d, 1<<8
    jg .hv_loop0
    RET

cglobal warp_affine_8x8t_8bpc, 4, 7, 22, tmp, ts
    vpbroadcastd         m9, [pd_16384]
    mova               ym15, [warp_8x8t_end]
    call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main
    jmp .start
.loop:
    call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx512icl).main2
    lea                tmpq, [tmpq+tsq*4]
.start:
    paddd               m16, m16
    vpermb              m16, m15, m16
    mova         [tmpq+tsq*0], xm16
    vextracti128 [tmpq+tsq*2], ym16, 1
    sub                 r6d, 0x1800
    jg .loop
    RET

cglobal warp_affine_8x8_8bpc, 4, 7, 22, dst, ds, src, ss, abcd, filter
    vpbroadcastd         m9, [pd_262144]
    mova               xm15, [warp_8x8_end]
    call .main
    jmp .start
.loop:
    call .main2
    lea                dstq, [dstq+dsq*2]
.start:
    psrad               m16, 19
    packuswb            m16, m16
    vpermb              m16, m15, m16
    movq       [dstq+dsq*0], xm16
    movhps     [dstq+dsq*1], xm16
    sub                 r6d, 0x1800
    jg .loop
    RET
ALIGN function_align
.main:
    vpbroadcastd         m1, [pd_512]
%if WIN64
    mov               abcdq, r5mp
    vpaddd             ym18, ym1, r6m {1to8} ; mx
%else
    add                 r5d, 512
    vpbroadcastd       ym18, r5d
%endif
    vpaddd             ym20, ym1, r7m {1to8} ; my
    mova               ym16, [pd_0to7]
    vpbroadcastd       ym19, [abcdq+4*0]
    vpbroadcastd       ym21, [abcdq+4*1]
    lea                  r4, [ssq*3+3]
    mova                m10, [warp_8x8_permA]
    mov                 r6d, 0x5555
    mova                m11, [warp_8x8_permB]
    lea             filterq, [mc_warp_filter+64*8]
    vpbroadcastq        m12, [warp_8x8_hpack]
    sub                srcq, r4               ; src -= src_stride*3 + 3
    vbroadcasti32x4     m13, [warp_8x8_permC]
    kxnorb               k2, k2, k2
    vbroadcasti32x4     m14, [warp_8x8_permD]
    vpdpwssd           ym18, ym19, ym16       ; alpha
    vpdpwssd           ym20, ym21, ym16       ; gamma
    vbroadcasti32x4      m0, [srcq]
    psrad              ym19, 16               ; beta
    psrad              ym21, 16               ; delta
    kmovw                k1, r6d
    psrad              ym16, ym18, 10
    kmovb                k3, k2
    paddd              ym18, ym19
    vpgatherdq       m2{k2}, [filterq+ym16*8] ; filter_x0
    psrld                m1, 8                ; pd_2
    pshufb               m0, m11
    paddd                m8, m1, m1           ; pd_4
    vpdpbusd             m1, m0, m2
    call .h
    psllq                m2, m1, 45
    pslld                m1, 13
    paddd                m1, m2
    vpshrdq              m1, m0, 48           ; 01 12
    call .h
    vpshrdq              m2, m1, m0, 48       ; 23 34
    call .h
    vpshrdq              m3, m2, m0, 48       ; 45 56
.main2:
    call .h
    psrad              ym17, ym20, 10
    kmovb                k2, k3
    paddd              ym20, ym21
    vpgatherdq       m7{k3}, [filterq+ym17*8] ; filter_y0
    psrad              ym16, ym20, 10
    kmovb                k3, k2
    paddd              ym20, ym21
    vpgatherdq      m17{k2}, [filterq+ym16*8] ; filter_y1
    shufps               m5, m7, m17, q2020   ; a0 a1 a2 a3 b0 b1 b2 b3 A0 A1 A2 A3 B0 B1 B2 B3
    mova                m16, m9
    pshufb               m4, m5, m13          ;    a0    a1    A0    A1    b0    b1    B0    B1
    vpdpwssd            m16, m1, m4
    pshufb               m5, m14              ;    a2    a3    A2    A3    b2    b3    B2    B3
    mova                 m1, m2
    vpdpwssd            m16, m2, m5
    shufps               m5, m7, m17, q3131   ; a4 a5 a6 a7 b4 b5 b6 b7 A4 A5 A6 A7 B4 B5 B6 B7
    mova                 m2, m3
    pshufb               m4, m5, m13          ;    a4    a5    A4    A5    b4    b5    B4    B5
    vpdpwssd            m16, m3, m4
    vpshrdq              m3, m0, 48           ; 67 78
    pshufb               m5, m14              ;    a6    a7    A6    A7    b6    b7    B6    B7
    vpdpwssd            m16, m3, m5
    ret
ALIGN function_align
.h:
    movu                xm5, [srcq+ssq*1]
    psrad              ym16, ym18, 10
    lea                srcq, [srcq+ssq*2]
    vinserti32x4        ym5, [srcq+ssq*0], 1
    kmovb                k2, k3
    paddd              ym18, ym19
    vpgatherdq       m6{k3}, [filterq+ym16*8] ; filter_x1
    psrad              ym17, ym18, 10
    kmovb                k3, k2
    paddd              ym18, ym19
    vpgatherdq      m16{k2}, [filterq+ym17*8] ; filter_x2
    mova                 m0, m8
    vpermb               m4, m10, m5          ; a4 b0 a5 b1   a6 b2 a7 b3   a8 b4 a9 b5   aa b6 ab b7
    vpshldq             m17, m16, m6, 32      ; a4 a5 a6 a7   b0 b1 b2 b3
    vpdpbusd             m0, m4, m17
    vpermb               m5, m11, m5          ; a0 b4 a1 b5   a2 b6 a3 b7   a4 b8 a5 b9   a6 ba a7 bb
    vmovdqa32       m16{k1}, m6               ; a0 a1 a2 a3   b4 b5 b6 b7
    vpdpbusd             m0, m5, m16
    vpmultishiftqb       m0, m12, m0          ; 1 1 2 2 (>> 3)
    ret

%macro BIDIR_FN 1 ; op
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    cmp                  hd, 8
    jg .w4_h16
    WRAP_YMM %1           0
    vextracti32x4       xm1, ym0, 1
    movd   [dstq          ], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    jl .w4_ret
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq          ], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
.w4_ret:
    RET
.w4_h16:
    vpbroadcastd         m7, strided
    pmulld               m7, [bidir_sctr_w4]
    %1                    0
    kxnorw               k1, k1, k1
    vpscatterdd [dstq+m7]{k1}, m0
    RET
.w8:
    cmp                  hd, 4
    jne .w8_h8
    WRAP_YMM %1           0
    vextracti32x4       xm1, ym0, 1
    movq   [dstq          ], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    RET
.w8_loop:
    %1_INC_PTR            2
    lea                dstq, [dstq+strideq*4]
.w8_h8:
    %1                    0
    vextracti32x4       xm1, ym0, 1
    vextracti32x4       xm2, m0, 2
    vextracti32x4       xm3, m0, 3
    movq   [dstq          ], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    movhps [dstq          ], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    sub                  hd, 8
    jg .w8_loop
    RET
.w16_loop:
    %1_INC_PTR            2
    lea                dstq, [dstq+strideq*4]
.w16:
    %1                    0
    vpermq               m0, m0, q3120
    mova          [dstq          ], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+stride3q ], m0, 3
    sub                  hd, 4
    jg .w16_loop
    RET
.w32:
    pmovzxbq             m7, [pb_02461357]
.w32_loop:
    %1                    0
    %1_INC_PTR            2
    vpermq               m0, m7, m0
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32_loop
    RET
.w64:
    pmovzxbq             m7, [pb_02461357]
.w64_loop:
    %1                    0
    %1_INC_PTR            2
    vpermq               m0, m7, m0
    mova             [dstq], m0
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET
.w128:
    pmovzxbq             m7, [pb_02461357]
.w128_loop:
    %1                    0
    vpermq               m6, m7, m0
    %1                    2
    mova        [dstq+64*0], m6
    %1_INC_PTR            4
    vpermq               m6, m7, m0
    mova        [dstq+64*1], m6
    add                dstq, strideq
    dec                  hd
    jg .w128_loop
    RET
%endmacro

%macro AVG 1 ; src_offset
    mova                 m0, [tmp1q+(%1+0)*mmsize]
    paddw                m0, [tmp2q+(%1+0)*mmsize]
    mova                 m1, [tmp1q+(%1+1)*mmsize]
    paddw                m1, [tmp2q+(%1+1)*mmsize]
    pmulhrsw             m0, m4
    pmulhrsw             m1, m4
    packuswb             m0, m1
%endmacro

%macro AVG_INC_PTR 1
    add               tmp1q, %1*mmsize
    add               tmp2q, %1*mmsize
%endmacro

cglobal avg_8bpc, 4, 7, 5, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-avg_avx512icl_table
    lea                  r6, [avg_avx512icl_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, dword [r6+wq*4]
    vpbroadcastd         m4, [base+pw_1024]
    add                  wq, r6
    BIDIR_FN            AVG

%macro W_AVG 1 ; src_offset
    ; (a * weight + b * (16 - weight) + 128) >> 8
    ; = ((a - b) * weight + (b << 4) + 128) >> 8
    ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4
    ; = ((((b - a) * (-weight     << 12)) >> 16) + b + 8) >> 4
    mova                 m0,     [tmp1q+(%1+0)*mmsize]
    psubw                m2, m0, [tmp2q+(%1+0)*mmsize]
    mova                 m1,     [tmp1q+(%1+1)*mmsize]
    psubw                m3, m1, [tmp2q+(%1+1)*mmsize]
    pmulhw               m2, m4
    pmulhw               m3, m4
    paddw                m0, m2
    paddw                m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
%endmacro

%define W_AVG_INC_PTR AVG_INC_PTR

cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3
%define base r6-w_avg_avx512icl_table
    lea                  r6, [w_avg_avx512icl_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    vpbroadcastw         m4, r6m ; weight
    movsxd               wq, dword [r6+wq*4]
    vpbroadcastd         m5, [base+pw_2048]
    psllw                m4, 12 ; (weight-16) << 12 when interpreted as signed
    add                  wq, r6
    cmp           dword r6m, 7
    jg .weight_gt7
    mov                  r6, tmp1q
    pxor                 m0, m0
    mov               tmp1q, tmp2q
    psubw                m4, m0, m4 ; -weight
    mov               tmp2q, r6
.weight_gt7:
    BIDIR_FN          W_AVG

%macro MASK 1 ; src_offset
    ; (a * m + b * (64 - m) + 512) >> 10
    ; = ((a - b) * m + (b << 6) + 512) >> 10
    ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4
%if mmsize == 64
    vpermq               m3, m8, [maskq+%1*32]
%else
    vpermq               m3,     [maskq+%1*16], q3120
%endif
    mova                 m0,     [tmp2q+(%1+0)*mmsize]
    psubw                m1, m0, [tmp1q+(%1+0)*mmsize]
    psubb                m3, m4, m3
    paddw                m1, m1     ; (b - a) << 1
    paddb                m3, m3
    punpcklbw            m2, m4, m3 ; -m << 9
    pmulhw               m1, m2
    paddw                m0, m1
    mova                 m1,     [tmp2q+(%1+1)*mmsize]
    psubw                m2, m1, [tmp1q+(%1+1)*mmsize]
    paddw                m2, m2
    punpckhbw            m3, m4, m3
    pmulhw               m2, m3
    paddw                m1, m2
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
%endmacro

%macro MASK_INC_PTR 1
    add               maskq, %1*32
    add               tmp2q, %1*64
    add               tmp1q, %1*64
%endmacro

cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-mask_avx512icl_table
    lea                  r7, [mask_avx512icl_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    mov               maskq, maskmp
    movsxd               wq, dword [r7+wq*4]
    pxor                 m4, m4
    mova                 m8, [base+bilin_v_perm64]
    vpbroadcastd         m5, [base+pw_2048]
    add                  wq, r7
    BIDIR_FN           MASK

%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4
    mova                m%1, [tmp1q+mmsize*%3]
    mova                 m1, [tmp2q+mmsize*%3]
    psubw                m1, m%1
    pabsw               m%2, m1
    psubusw             m%2, m6, m%2
    psrlw               m%2, 8 ; 64 - m
    psllw                m2, m%2, 10
    pmulhw               m1, m2
    paddw               m%1, m1
    mova                 m1, [tmp1q+mmsize*%4]
    mova                 m2, [tmp2q+mmsize*%4]
    psubw                m2, m1
    pabsw                m3, m2
    psubusw              m3, m6, m3
    vpshldw             m%2, m3, 8
    psllw                m3, m%2, 10
%if %5
    psubb               m%2, m5, m%2
%endif
    pmulhw               m2, m3
    paddw                m1, m2
    pmulhrsw            m%1, m7
    pmulhrsw             m1, m7
    packuswb            m%1, m1
%endmacro

cglobal w_mask_420_8bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_420_avx512icl_table
    lea                  r7, [w_mask_420_avx512icl_table]
    tzcnt                wd, wm
    mov                 r6d, r7m ; sign
    movifnidn            hd, hm
    movsxd               wq, [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m7, [base+pw_2048]
    vpbroadcastd         m9, [base+pb_m64]             ; -1 << 6
    mova               ym10, [base+wm_420_mask+32]
    vpbroadcastd         m8, [base+wm_sign+r6*8] ; (258 - sign) << 6
    add                  wq, r7
    mov               maskq, maskmp
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    mova                 m5, [wm_420_perm4]
    cmp                  hd, 8
    jg .w4_h16
    WRAP_YMM W_MASK       0, 4, 0, 1
    vinserti128         ym5, [wm_420_perm4+32], 1
    vpermb              ym4, ym5, ym4
    vpdpbusd            ym8, ym4, ym9
    vextracti32x4       xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
.w4_end:
    vpermb              ym8, ym10, ym8
    movq            [maskq], xm8
    RET
.w4_h16:
    vpbroadcastd        m11, strided
    pmulld              m11, [bidir_sctr_w4]
    W_MASK                0, 4, 0, 1
    vpermb               m4, m5, m4
    vpdpbusd             m8, m4, m9
    kxnorw               k1, k1, k1
    vpermb               m8, m10, m8
    mova            [maskq], xm8
    vpscatterdd [dstq+m11]{k1}, m0
    RET
.w8:
    mova                 m5, [wm_420_perm8]
    cmp                  hd, 4
    jne .w8_h8
    WRAP_YMM W_MASK       0, 4, 0, 1
    vinserti128         ym5, [wm_420_perm8+32], 1
    vpermb              ym4, ym5, ym4
    vpdpbusd            ym8, ym4, ym9
    vpermb               m8, m10, m8
    mova            [maskq], xm8
    vextracti32x4       xm1, ym0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    RET
.w8_loop:
    add               tmp1q, 128
    add               tmp2q, 128
    add               maskq, 16
    lea                dstq, [dstq+strideq*4]
.w8_h8:
    W_MASK                0, 4, 0, 1
    vpermb               m4, m5, m4
    mova                 m1, m8
    vpdpbusd             m1, m4, m9
    vpermb               m1, m10, m1
    mova            [maskq], xm1
    vextracti32x4       xm1, ym0, 1
    vextracti32x4       xm2, m0, 2
    vextracti32x4       xm3, m0, 3
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    sub                  hd, 8
    jg .w8_loop
    RET
.w16:
    mova                 m5, [wm_420_perm16]
.w16_loop:
    W_MASK                0, 4, 0, 1
    vpermb               m4, m5, m4
    mova                 m1, m8
    vpdpbusd             m1, m4, m9
    add               tmp1q, 128
    add               tmp2q, 128
    vpermb               m1, m10, m1
    vpermq               m0, m0, q3120
    mova            [maskq], xm1
    add               maskq, 16
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+stride3q ], m0, 3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w16_loop
    RET
.w32:
    pmovzxbq             m5, [pb_02461357]
.w32_loop:
    W_MASK                0, 4, 0, 1
    mova                 m1, m8
    vpdpbusd             m1, m4, m9
    add               tmp1q, 128
    add               tmp2q, 128
    vpermb               m1, m10, m1
    vpermq               m0, m5, m0
    mova            [maskq], xm1
    add               maskq, 16
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32_loop
    RET
.w64:
    pmovzxbq            m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14
    psrlq               m13, m12, 4          ; 1, 3, 5, 7, 9, 11, 13, 15
.w64_loop:
    W_MASK                0, 4, 0, 2
    W_MASK               11, 5, 1, 3
    mova                 m2, m8
    vpdpbusd             m2, m4, m9
    mova                 m3, m8
    vpdpbusd             m3, m5, m9
    add               tmp1q, 256
    add               tmp2q, 256
    vpermt2b             m2, m10, m3
    mova                 m1, m0
    vpermt2q             m0, m12, m11
    vpermt2q             m1, m13, m11
    mova            [maskq], ym2
    add               maskq, 32
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w64_loop
    RET
.w128:
    pmovzxbq            m14, [wm_420_perm64]
    mova                m10, [wm_420_mask]
    psrlq               m15, m14, 4
.w128_loop:
    W_MASK                0, 12, 0, 4
    W_MASK               11, 13, 1, 5
    mova                 m4, m8
    vpdpbusd             m4, m12, m9
    mova                 m5, m8
    vpdpbusd             m5, m13, m9
    mova                 m1, m0
    vpermt2q             m0, m14, m11
    vpermt2q             m1, m15, m11
    mova [dstq+strideq*0+64*0], m0
    mova [dstq+strideq*1+64*0], m1
    W_MASK                0, 12, 2, 6
    W_MASK               11, 13, 3, 7
    vprold               m4, 16
    vprold               m5, 16
    vpdpbusd             m4, m12, m9
    vpdpbusd             m5, m13, m9
    add               tmp1q, 512
    add               tmp2q, 512
    vpermt2b             m4, m10, m5
    mova                 m1, m0
    vpermt2q             m0, m14, m11
    vpermt2q             m1, m15, m11
    mova            [maskq], m4
    add               maskq, 64
    mova [dstq+strideq*0+64*1], m0
    mova [dstq+strideq*1+64*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w128_loop
    RET

cglobal w_mask_422_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_422_avx512icl_table
    lea                  r7, [w_mask_422_avx512icl_table]
    tzcnt                wd, wm
    mov                 r6d, r7m ; sign
    movifnidn            hd, hm
    movsxd               wq, dword [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m7, [base+pw_2048]
    vpbroadcastd         m9, [base+pw_m128]
    mova                m10, [base+wm_422_mask]
    vpbroadcastd        m11, [base+pb_127]
    add                  wq, r7
    vpbroadcastd         m8, [base+wm_sign+4+r6*4]
    mov               maskq, maskmp
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    cmp                  hd, 8
    jg .w4_h16
    WRAP_YMM W_MASK       0, 4, 0, 1
    movhps             xm10, [wm_422_mask+16]
    vpdpwssd            ym8, ym4, ym9
    vpermb              ym8, ym10, ym8
    vextracti32x4       xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
.w4_end:
    pand                xm8, xm11
    mova            [maskq], xm8
    RET
.w4_h16:
    vpbroadcastd         m5, strided
    pmulld               m5, [bidir_sctr_w4]
    W_MASK                0, 4, 0, 1
    vpdpwssd             m8, m4, m9
    kxnorw               k1, k1, k1
    vpermb               m8, m10, m8
    pand                ym8, ym11
    mova            [maskq], ym8
    vpscatterdd [dstq+m5]{k1}, m0
    RET
.w8:
    cmp                  hd, 4
    jne .w8_h8
    WRAP_YMM W_MASK       0, 4, 0, 1
    movhps             xm10, [wm_422_mask+16]
    vpdpwssd            ym8, ym4, ym9
    vpermb              ym8, ym10, ym8
    pand                xm8, xm11
    mova            [maskq], xm8
    vextracti32x4       xm1, ym0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    RET
.w8_loop:
    add               tmp1q, 128
    add               tmp2q, 128
    add               maskq, 32
    lea                dstq, [dstq+strideq*4]
.w8_h8:
    W_MASK                0, 4, 0, 1
    mova                 m1, m8
    vpdpwssd             m1, m4, m9
    vpermb               m1, m10, m1
    pand                ym1, ym11
    mova            [maskq], ym1
    vextracti32x4       xm1, ym0, 1
    vextracti32x4       xm2, m0, 2
    vextracti32x4       xm3, m0, 3
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    sub                  hd, 8
    jg .w8_loop
    RET
.w16_loop:
    add               tmp1q, 128
    add               tmp2q, 128
    add               maskq, 32
    lea                dstq, [dstq+strideq*4]
.w16:
    W_MASK                0, 4, 0, 1
    mova                 m1, m8
    vpdpwssd             m1, m4, m9
    vpermb               m1, m10, m1
    vpermq               m0, m0, q3120
    pand                ym1, ym11
    mova            [maskq], ym1
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+stride3q ], m0, 3
    sub                  hd, 4
    jg .w16_loop
    RET
.w32:
    pmovzxbq             m5, [pb_02461357]
.w32_loop:
    W_MASK                0, 4, 0, 1
    mova                 m1, m8
    vpdpwssd             m1, m4, m9
    add               tmp1q, 128
    add               tmp2q, 128
    vpermb               m1, m10, m1
    vpermq               m0, m5, m0
    pand                ym1, ym11
    mova            [maskq], ym1
    add               maskq, 32
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32_loop
    RET
.w64:
    pmovzxbq             m5, [pb_02461357]
.w64_loop:
    W_MASK                0, 4, 0, 1
    mova                 m1, m8
    vpdpwssd             m1, m4, m9
    add               tmp1q, 128
    add               tmp2q, 128
    vpermb               m1, m10, m1
    vpermq               m0, m5, m0
    pand                ym1, ym11
    mova            [maskq], ym1
    add               maskq, 32
    mova             [dstq], m0
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET
.w128:
    pmovzxbq            m13, [pb_02461357]
.w128_loop:
    W_MASK                0, 4, 0, 1
    W_MASK               12, 5, 2, 3
    mova                 m2, m8
    vpdpwssd             m2, m4, m9
    mova                 m3, m8
    vpdpwssd             m3, m5, m9
    add               tmp1q, 256
    add               tmp2q, 256
    vpermt2b             m2, m10, m3
    vpermq               m0, m13, m0
    vpermq               m1, m13, m12
    pand                 m2, m11
    mova            [maskq], m2
    add               maskq, 64
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m1
    add                dstq, strideq
    dec                  hd
    jg .w128_loop
    RET

cglobal w_mask_444_8bpc, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3
%define base r7-w_mask_444_avx512icl_table
    lea                  r7, [w_mask_444_avx512icl_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, dword [r7+wq*4]
    vpbroadcastd         m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8
    vpbroadcastd         m5, [base+pb_64]
    vpbroadcastd         m7, [base+pw_2048]
    mova                 m8, [base+wm_444_mask]
    add                  wq, r7
    mov               maskq, maskmp
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    cmp                  hd, 8
    jg .w4_h16
    WRAP_YMM W_MASK       0, 4, 0, 1, 1
    vinserti128         ym8, [wm_444_mask+32], 1
    vpermb              ym4, ym8, ym4
    mova            [maskq], ym4
    vextracti32x4      xm1, m0, 1
    movd   [dstq+strideq*0], xm0
    pextrd [dstq+strideq*1], xm0, 1
    movd   [dstq+strideq*2], xm1
    pextrd [dstq+stride3q ], xm1, 1
    jl .w4_end
    lea                dstq, [dstq+strideq*4]
    pextrd [dstq+strideq*0], xm0, 2
    pextrd [dstq+strideq*1], xm0, 3
    pextrd [dstq+strideq*2], xm1, 2
    pextrd [dstq+stride3q ], xm1, 3
.w4_end:
    RET
.w4_h16:
    vpbroadcastd         m9, strided
    pmulld               m9, [bidir_sctr_w4]
    W_MASK                0, 4, 0, 1, 1
    vpermb               m4, m8, m4
    kxnorw               k1, k1, k1
    mova            [maskq], m4
    vpscatterdd [dstq+m9]{k1}, m0
    RET
.w8:
    cmp                  hd, 4
    jne .w8_h8
    WRAP_YMM W_MASK       0, 4, 0, 1, 1
    vinserti128         ym8, [wm_444_mask+32], 1
    vpermb              ym4, ym8, ym4
    mova            [maskq], ym4
    vextracti32x4       xm1, ym0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    RET
.w8_loop:
    add               tmp1q, 128
    add               tmp2q, 128
    add               maskq, 64
    lea                dstq, [dstq+strideq*4]
.w8_h8:
    W_MASK                0, 4, 0, 1, 1
    vpermb               m4, m8, m4
    mova            [maskq], m4
    vextracti32x4       xm1, ym0, 1
    vextracti32x4       xm2, m0, 2
    vextracti32x4       xm3, m0, 3
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    sub                  hd, 8
    jg .w8_loop
    RET
.w16_loop:
    add               tmp1q, 128
    add               tmp2q, 128
    add               maskq, 64
    lea                dstq, [dstq+strideq*4]
.w16:
    W_MASK                0, 4, 0, 1, 1
    vpermb               m4, m8, m4
    vpermq               m0, m0, q3120
    mova            [maskq], m4
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+stride3q ], m0, 3
    sub                  hd, 4
    jg .w16_loop
    RET
.w32:
    pmovzxbq             m9, [pb_02461357]
.w32_loop:
    W_MASK                0, 4, 0, 1, 1
    vpermb               m4, m8, m4
    add               tmp1q, 128
    add               tmp2q, 128
    vpermq               m0, m9, m0
    mova            [maskq], m4
    add               maskq, 64
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32_loop
    RET
.w64:
    pmovzxbq             m9, [pb_02461357]
.w64_loop:
    W_MASK                0, 4, 0, 1, 1
    vpermb               m4, m8, m4
    add               tmp1q, 128
    add               tmp2q, 128
    vpermq               m0, m9, m0
    mova            [maskq], m4
    add               maskq, 64
    mova             [dstq], m0
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET
.w128:
    pmovzxbq            m11, [pb_02461357]
.w128_loop:
    W_MASK                0, 4, 0, 1, 1
    W_MASK               10, 9, 2, 3, 1
    vpermb               m4, m8, m4
    vpermb               m9, m8, m9
    add               tmp1q, 256
    add               tmp2q, 256
    vpermq               m0, m11, m0
    vpermq              m10, m11, m10
    mova       [maskq+64*0], m4
    mova       [maskq+64*1], m9
    add               maskq, 128
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m10
    add                dstq, strideq
    dec                  hd
    jg .w128_loop
    RET

cglobal blend_8bpc, 3, 7, 8, dst, ds, tmp, w, h, mask
%define base r6-blend_avx512icl_table
    lea                  r6, [blend_avx512icl_table]
    tzcnt                wd, wm
    movifnidn         maskq, maskmp
    movifnidn            hd, hm
    movsxd               wq, [r6+wq*4]
    vpbroadcastd         m6, [base+pb_64]
    vpbroadcastd         m7, [base+pw_512]
    sub                tmpq, maskq
    add                  wq, r6
    lea                  r6, [dsq*3]
    jmp                  wq
.w4:
    movd               xmm0, [dstq+dsq*0]
    pinsrd             xmm0, [dstq+dsq*1], 1
    vpbroadcastd       xmm1, [dstq+dsq*2]
    pinsrd             xmm1, [dstq+r6   ], 3
    mova               xmm4, [maskq]
    mova               xmm5, [maskq+tmpq]
    add               maskq, 4*4
    psubb              xmm3, xm6, xmm4
    punpcklbw          xmm0, xmm5
    punpcklbw          xmm2, xmm3, xmm4
    punpckhbw          xmm1, xmm5
    punpckhbw          xmm3, xmm4
    pmaddubsw          xmm0, xmm2
    pmaddubsw          xmm1, xmm3
    pmulhrsw           xmm0, xm7
    pmulhrsw           xmm1, xm7
    packuswb           xmm0, xmm1
    movd       [dstq+dsq*0], xmm0
    pextrd     [dstq+dsq*1], xmm0, 1
    pextrd     [dstq+dsq*2], xmm0, 2
    pextrd     [dstq+r6   ], xmm0, 3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    movq               xmm0, [dstq+dsq*0]
    vpbroadcastq       xmm1, [dstq+dsq*1]
    vpbroadcastq       ymm2, [dstq+dsq*2]
    vpbroadcastq       ymm3, [dstq+r6   ]
    mova               ymm4, [maskq]
    mova               ymm5, [maskq+tmpq]
    add               maskq, 8*4
    vpblendd           ymm0, ymm2, 0x30
    vpblendd           ymm1, ymm3, 0xc0
    psubb              ymm3, ym6, ymm4
    punpcklbw          ymm0, ymm5
    punpcklbw          ymm2, ymm3, ymm4
    punpckhbw          ymm1, ymm5
    punpckhbw          ymm3, ymm4
    pmaddubsw          ymm0, ymm2
    pmaddubsw          ymm1, ymm3
    pmulhrsw           ymm0, ym7
    pmulhrsw           ymm1, ym7
    packuswb           ymm0, ymm1
    vextracti128       xmm1, ymm0, 1
    movq       [dstq+dsq*0], xmm0
    movhps     [dstq+dsq*1], xmm0
    movq       [dstq+dsq*2], xmm1
    movhps     [dstq+r6   ], xmm1
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 4
    jg .w8
    vzeroupper
    RET
.w16:
    mova                xm1, [dstq+dsq*0]
    vinserti32x4        ym1, [dstq+dsq*1], 1
    vinserti32x4         m1, [dstq+dsq*2], 2
    mova                 m4, [maskq]
    vinserti32x4         m1, [dstq+r6   ], 3
    mova                 m5, [maskq+tmpq]
    add               maskq, 16*4
    psubb                m3, m6, m4
    punpcklbw            m0, m1, m5
    punpcklbw            m2, m3, m4
    punpckhbw            m1, m5
    punpckhbw            m3, m4
    pmaddubsw            m0, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m7
    pmulhrsw             m1, m7
    packuswb             m0, m1
    mova          [dstq+dsq*0], xm0
    vextracti32x4 [dstq+dsq*1], ym0, 1
    vextracti32x4 [dstq+dsq*2], m0, 2
    vextracti32x4 [dstq+r6   ], m0, 3
    lea                dstq, [dstq+dsq*4]
    sub                  hd, 4
    jg .w16
    RET
.w32:
    mova                ym1, [dstq+dsq*0]
    vinserti32x8         m1, [dstq+dsq*1], 1
    mova                 m4, [maskq]
    mova                 m5, [maskq+tmpq]
    add               maskq, 32*2
    psubb                m3, m6, m4
    punpcklbw            m0, m1, m5
    punpcklbw            m2, m3, m4
    punpckhbw            m1, m5
    punpckhbw            m3, m4
    pmaddubsw            m0, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m7
    pmulhrsw             m1, m7
    packuswb             m0, m1
    mova          [dstq+dsq*0], ym0
    vextracti32x8 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w32
    RET

cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask
%define base r5-blend_v_avx512icl_table
    lea                  r5, [blend_v_avx512icl_table]
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, [r5+wq*4]
    vpbroadcastd         m5, [base+pw_512]
    add                  wq, r5
    add               maskq, obmc_masks-blend_v_avx512icl_table
    jmp                  wq
.w2:
    vpbroadcastd       xmm2, [maskq+2*2]
.w2_s0_loop:
    movd               xmm0, [dstq+dsq*0]
    pinsrw             xmm0, [dstq+dsq*1], 1
    movd               xmm1, [tmpq]
    add                tmpq, 2*2
    punpcklbw          xmm0, xmm1
    pmaddubsw          xmm0, xmm2
    pmulhrsw           xmm0, xm5
    packuswb           xmm0, xmm0
    pextrw     [dstq+dsq*0], xmm0, 0
    pextrw     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w2_s0_loop
    RET
.w4:
    vpbroadcastq       xmm2, [maskq+4*2]
.w4_loop:
    movd               xmm0, [dstq+dsq*0]
    pinsrd             xmm0, [dstq+dsq*1], 1
    movq               xmm1, [tmpq]
    add                tmpq, 4*2
    punpcklbw          xmm0, xmm1
    pmaddubsw          xmm0, xmm2
    pmulhrsw           xmm0, xm5
    packuswb           xmm0, xmm0
    movd       [dstq+dsq*0], xmm0
    pextrd     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w4_loop
    RET
.w8:
    mova               xmm3, [maskq+8*2]
.w8_loop:
    movq               xmm0, [dstq+dsq*0]
    vpbroadcastq       xmm1, [dstq+dsq*1]
    mova               xmm2, [tmpq]
    add                tmpq, 8*2
    punpcklbw          xmm0, xmm2
    punpckhbw          xmm1, xmm2
    pmaddubsw          xmm0, xmm3
    pmaddubsw          xmm1, xmm3
    pmulhrsw           xmm0, xm5
    pmulhrsw           xmm1, xm5
    packuswb           xmm0, xmm1
    movq       [dstq+dsq*0], xmm0
    movhps     [dstq+dsq*1], xmm0
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w8_loop
    RET
.w16:
    vbroadcasti32x4     ym3, [maskq+16*2]
    vbroadcasti32x4     ym4, [maskq+16*3]
.w16_loop:
    mova                xm1, [dstq+dsq*0]
    vinserti32x4        ym1, [dstq+dsq*1], 1
    mova                ym2, [tmpq]
    add                tmpq, 16*2
    punpcklbw           ym0, ym1, ym2
    punpckhbw           ym1, ym2
    pmaddubsw           ym0, ym3
    pmaddubsw           ym1, ym4
    pmulhrsw            ym0, ym5
    pmulhrsw            ym1, ym5
    packuswb            ym0, ym1
    mova          [dstq+dsq*0], xm0
    vextracti32x4 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w16_loop
    RET
.w32:
    mova                 m4, [maskq+32*2]
    vshufi32x4           m3, m4, m4, q2020
    vshufi32x4           m4, m4, q3131
.w32_loop:
    mova                ym1, [dstq+dsq*0]
    vinserti32x8         m1, [dstq+dsq*1], 1
    mova                 m2, [tmpq]
    add                tmpq, 32*2
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    pmaddubsw            m0, m3
    pmaddubsw            m1, m4
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    mova          [dstq+dsq*0], ym0
    vextracti32x8 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    sub                  hd, 2
    jg .w32_loop
    RET

cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask
%define base r6-blend_h_avx512icl_table
    lea                  r6, [blend_h_avx512icl_table]
    tzcnt                wd, wm
    mov                  hd, hm
    movsxd               wq, [r6+wq*4]
    lea               maskq, [base+obmc_masks+hq*2]
    vpbroadcastd         m5, [base+pw_512]
    lea                  hd, [hq*3]
    add                  wq, r6
    shr                  hd, 2 ; h * 3/4
    lea               maskq, [maskq+hq*2]
    neg                  hq
    jmp                  wq
.w2:
    movd               xmm0, [dstq+dsq*0]
    pinsrw             xmm0, [dstq+dsq*1], 1
    movd               xmm2, [maskq+hq*2]
    movd               xmm1, [tmpq]
    add                tmpq, 2*2
    punpcklwd          xmm2, xmm2
    punpcklbw          xmm0, xmm1
    pmaddubsw          xmm0, xmm2
    pmulhrsw           xmm0, xm5
    packuswb           xmm0, xmm0
    pextrw     [dstq+dsq*0], xmm0, 0
    pextrw     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    add                  hq, 2
    jl .w2
    RET
.w4:
    mova               xmm3, [blend_shuf]
.w4_loop:
    movd               xmm0, [dstq+dsq*0]
    pinsrd             xmm0, [dstq+dsq*1], 1
    movd               xmm2, [maskq+hq*2]
    movq               xmm1, [tmpq]
    add                tmpq, 4*2
    pshufb             xmm2, xmm3
    punpcklbw          xmm0, xmm1
    pmaddubsw          xmm0, xmm2
    pmulhrsw           xmm0, xm5
    packuswb           xmm0, xmm0
    movd       [dstq+dsq*0], xmm0
    pextrd     [dstq+dsq*1], xmm0, 1
    lea                dstq, [dstq+dsq*2]
    add                  hq, 2
    jl .w4_loop
    RET
.w8:
    vbroadcasti128     ymm4, [blend_shuf]
    shufpd             ymm4, ymm4, 0x03
.w8_loop:
    vpbroadcastq       ymm1, [dstq+dsq*0]
    movq               xmm0, [dstq+dsq*1]
    vpblendd           ymm0, ymm1, 0x30
    vpbroadcastd       ymm3, [maskq+hq*2]
    movq               xmm1, [tmpq+8*1]
    vinserti128        ymm1, [tmpq+8*0], 1
    add                tmpq, 8*2
    pshufb             ymm3, ymm4
    punpcklbw          ymm0, ymm1
    pmaddubsw          ymm0, ymm3
    pmulhrsw           ymm0, ym5
    vextracti128       xmm1, ymm0, 1
    packuswb           xmm0, xmm1
    movhps     [dstq+dsq*0], xmm0
    movq       [dstq+dsq*1], xmm0
    lea                dstq, [dstq+dsq*2]
    add                  hq, 2
    jl .w8_loop
    vzeroupper
    RET
.w16:
    vbroadcasti32x4     ym4, [blend_shuf]
    shufpd              ym4, ym4, 0x0c
.w16_loop:
    mova                xm1, [dstq+dsq*0]
    vinserti32x4        ym1, [dstq+dsq*1], 1
    vpbroadcastd        ym3, [maskq+hq*2]
    mova                ym2, [tmpq]
    add                tmpq, 16*2
    pshufb              ym3, ym4
    punpcklbw           ym0, ym1, ym2
    punpckhbw           ym1, ym2
    pmaddubsw           ym0, ym3
    pmaddubsw           ym1, ym3
    pmulhrsw            ym0, ym5
    pmulhrsw            ym1, ym5
    packuswb            ym0, ym1
    mova          [dstq+dsq*0], xm0
    vextracti32x4 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    add                  hq, 2
    jl .w16_loop
    RET
.w32:
    vbroadcasti32x4      m4, [blend_shuf]
    shufpd               m4, m4, 0xf0
.w32_loop:
    mova                ym1, [dstq+dsq*0]
    vinserti32x8         m1, [dstq+dsq*1], 1
    vpbroadcastd         m3, [maskq+hq*2]
    mova                 m2, [tmpq]
    add                tmpq, 32*2
    pshufb               m3, m4
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    pmaddubsw            m0, m3
    pmaddubsw            m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    mova          [dstq+dsq*0], ym0
    vextracti32x8 [dstq+dsq*1], m0, 1
    lea                dstq, [dstq+dsq*2]
    add                  hq, 2
    jl .w32_loop
    RET
.w64:
    vpbroadcastw         m3, [maskq+hq*2]
    mova                 m1, [dstq]
    mova                 m2, [tmpq]
    add                tmpq, 32*2
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    pmaddubsw            m0, m3
    pmaddubsw            m1, m3
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    packuswb             m0, m1
    mova             [dstq], m0
    add                dstq, dsq
    inc                  hq
    jl .w64
    RET
.w128:
    vpbroadcastw         m6, [maskq+hq*2]
    mova                 m2, [dstq+64*0]
    mova                 m1, [tmpq+64*0]
    mova                 m3, [dstq+64*1]
    mova                 m4, [tmpq+64*1]
    add                tmpq, 64*2
    punpcklbw            m0, m2, m1
    punpckhbw            m2, m1
    pmaddubsw            m0, m6
    pmaddubsw            m2, m6
    punpcklbw            m1, m3, m4
    punpckhbw            m3, m4
    pmaddubsw            m1, m6
    pmaddubsw            m3, m6
    REPX   {pmulhrsw x, m5}, m0, m2, m1, m3
    packuswb             m0, m2
    packuswb             m1, m3
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m1
    add                dstq, dsq
    inc                  hq
    jl .w128
    RET

cglobal resize_8bpc, 6, 12, 19, dst, dst_stride, src, src_stride, \
                                dst_w, h, src_w, dx, mx0
    sub          dword mx0m, 4<<14
    sub        dword src_wm, 8
    mov                  r6, ~0
    vpbroadcastd         m5, dxm
    vpbroadcastd         m8, mx0m
    vpbroadcastd         m6, src_wm
    kmovq                k3, r6
 DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x
    LEA                  r7, $$
%define base r7-$$
    vpbroadcastd         m3, [base+pw_m256]
    vpbroadcastd         m7, [base+pd_63]
    vbroadcasti32x4     m15, [base+pb_8x0_8x8]
    vpdpwssd             m8, m5, [base+rescale_mul] ; mx+dx*[0-15]
    pslld                m5, 4                      ; dx*16
    pslld                m6, 14
    pxor                 m2, m2
    mova                m16, [base+resize_permA]
    mova                m17, [base+resize_permB]
    mova               xm18, [base+resize_permC]
.loop_y:
    xor                  xd, xd
    mova                 m4, m8     ; per-line working version of mx
.loop_x:
    pmaxsd               m0, m4, m2
    psrad                m9, m4, 8  ; filter offset (unmasked)
    pminsd               m0, m6     ; iclip(mx, 0, src_w-8)
    psubd                m1, m4, m0 ; pshufb offset
    psrad                m0, 14     ; clipped src_x offset
    psrad                m1, 14     ; pshufb edge_emu offset
    vptestmd             k4, m1, m1
    pand                 m9, m7     ; filter offset (masked)
    ktestw               k4, k4
    jz .load
    vextracti32x8      ym12, m0, 1
    vextracti32x8      ym13, m1, 1
    kmovq                k1, k3
    kmovq                k2, k3
    vpgatherdq      m10{k1}, [srcq+ym0]
    vpgatherdq      m11{k2}, [srcq+ym12]
    kmovq                k1, k3
    kmovq                k2, k3
    vpgatherdq      m14{k1}, [base+resize_shuf+4+ym1]
    vpgatherdq       m0{k2}, [base+resize_shuf+4+ym13]
    mova                m12, m16
    mova                m13, m17
    paddb               m14, m15
    paddb                m0, m15
    pshufb              m10, m14
    pshufb              m11, m0
    vpermi2d            m12, m10, m11
    vpermi2d            m13, m10, m11
    jmp .filter
.load:
    kmovq                k1, k3
    kmovq                k2, k3
    vpgatherdd      m12{k1}, [srcq+m0+0]
    vpgatherdd      m13{k2}, [srcq+m0+4]
.filter:
    kmovq                k1, k3
    kmovq                k2, k3
    vpgatherdd      m10{k1}, [base+resize_filter+m9*8+0]
    vpgatherdd      m11{k2}, [base+resize_filter+m9*8+4]
    mova                m14, m2
    vpdpbusd            m14, m12, m10
    vpdpbusd            m14, m13, m11
    packssdw            m14, m14
    pmulhrsw            m14, m3
    packuswb            m14, m14
    vpermd              m14, m18, m14
    mova          [dstq+xq], xm14
    paddd                m4, m5
    add                  xd, 16
    cmp                  xd, dst_wd
    jl .loop_x
    add                dstq, dst_strideq
    add                srcq, src_strideq
    dec                  hd
    jg .loop_y
    RET

%endif ; ARCH_X86_64