; Copyright © 2022, VideoLAN and dav1d authors
; Copyright © 2022, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

%if ARCH_X86_64

SECTION_RODATA 64

ipred_shuf:    db 14, 15, 14, 15,  0,  1,  2,  3,  6,  7,  6,  7,  0,  1,  2,  3
               db 10, 11, 10, 11,  8,  9, 10, 11,  2,  3,  2,  3,  8,  9, 10, 11
               db 12, 13, 12, 13,  4,  5,  6,  7,  4,  5,  4,  5,  4,  5,  6,  7
               db  8,  9,  8,  9, 12, 13, 14, 15,  0,  1,  0,  1, 12, 13, 14, 15
smooth_perm:   db  1,  2,  5,  6,  9, 10, 13, 14, 17, 18, 21, 22, 25, 26, 29, 30
               db 33, 34, 37, 38, 41, 42, 45, 46, 49, 50, 53, 54, 57, 58, 61, 62
               db 65, 66, 69, 70, 73, 74, 77, 78, 81, 82, 85, 86, 89, 90, 93, 94
               db 97, 98,101,102,105,106,109,110,113,114,117,118,121,122,125,126
pal_pred_perm: db  0, 32,  1, 33,  2, 34,  3, 35,  4, 36,  5, 37,  6, 38,  7, 39
               db  8, 40,  9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47
               db 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55
               db 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63
filter_permA:  times 4 db  6,  7,  8,  9, 14, 15,  4,  5
               times 4 db 10, 11, 12, 13,  2,  3, -1, -1
filter_permB:  times 4 db 22, 23, 24, 25, 30, 31,  6,  7
               times 4 db 26, 27, 28, 29, 14, 15, -1, -1
filter_permC:          dd  8 ; dq  8, 10,  1, 11,  0,  9
pw_1:          times 2 dw  1
                       dd 10
filter_rnd:            dd 32
                       dd  1
                       dd  8
                       dd 11
filter_shift:  times 2 dw  6
                       dd  0
               times 2 dw  4
                       dd  9

%macro JMP_TABLE 3-*
    %xdefine %1_%2_table (%%table - 2*4)
    %xdefine %%base mangle(private_prefix %+ _%1_%2)
    %%table:
    %rep %0 - 2
        dd %%base %+ .%3 - (%%table - 2*4)
        %rotate 1
    %endrep
%endmacro

JMP_TABLE ipred_paeth_16bpc,      avx512icl, w4, w8, w16, w32, w64
JMP_TABLE ipred_smooth_16bpc,     avx512icl, w4, w8, w16, w32, w64
JMP_TABLE ipred_smooth_h_16bpc,   avx512icl, w4, w8, w16, w32, w64
JMP_TABLE ipred_smooth_v_16bpc,   avx512icl, w4, w8, w16, w32, w64
JMP_TABLE pal_pred_16bpc,         avx512icl, w4, w8, w16, w32, w64

cextern smooth_weights_1d_16bpc
cextern smooth_weights_2d_16bpc
cextern filter_intra_taps

SECTION .text

%macro PAETH 3 ; top, signed_ldiff, ldiff
    paddw               m0, m%2, m2
    psubw               m1, m0, m3  ; tldiff
    psubw               m0, m%1     ; tdiff
    pabsw               m1, m1
    pabsw               m0, m0
    pcmpgtw             k1, m0, m1
    pminsw              m0, m1
    pcmpgtw             k2, m%3, m0
    vpblendmw       m0{k1}, m%1, m3
    vpblendmw       m0{k2}, m2, m0
%endmacro

INIT_ZMM avx512icl
cglobal ipred_paeth_16bpc, 3, 7, 10, dst, stride, tl, w, h
%define base r6-ipred_paeth_16bpc_avx512icl_table
    lea                 r6, [ipred_paeth_16bpc_avx512icl_table]
    tzcnt               wd, wm
    movifnidn           hd, hm
    movsxd              wq, [r6+wq*4]
    vpbroadcastw        m3, [tlq]   ; topleft
    add                 wq, r6
    jmp                 wq
.w4:
    vpbroadcastq        m4, [tlq+2] ; top
    movsldup            m7, [base+ipred_shuf]
    lea                 r6, [strideq*3]
    psubw               m5, m4, m3
    pabsw               m6, m5
.w4_loop:
    sub                tlq, 16
    vbroadcasti32x4     m2, [tlq]
    pshufb              m2, m7      ; left
    PAETH                4, 5, 6
    vextracti32x4      xm1, m0, 2
    vextracti32x4      xm8, ym0, 1
    vextracti32x4      xm9, m0, 3
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm8
    movq   [dstq+r6       ], xm9
    sub                 hd, 8
    jl .w4_end
    lea               dstq, [dstq+strideq*4]
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm8
    movhps [dstq+r6       ], xm9
    lea               dstq, [dstq+strideq*4]
    jg .w4_loop
.w4_end:
    RET
.w8:
    vbroadcasti32x4     m4, [tlq+2]
    movsldup            m7, [base+ipred_shuf]
    lea                 r6, [strideq*3]
    psubw               m5, m4, m3
    pabsw               m6, m5
.w8_loop:
    sub                tlq, 8
    vpbroadcastq        m2, [tlq]
    pshufb              m2, m7
    PAETH                4, 5, 6
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+r6       ], m0, 3
    lea               dstq, [dstq+strideq*4]
    sub                 hd, 4
    jg .w8_loop
    RET
.w16:
    vbroadcasti32x8     m4, [tlq+2]
    movsldup            m7, [base+ipred_shuf]
    psubw               m5, m4, m3
    pabsw               m6, m5
.w16_loop:
    sub                tlq, 4
    vpbroadcastd        m2, [tlq]
    pshufb              m2, m7
    PAETH                4, 5, 6
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea               dstq, [dstq+strideq*2]
    sub                 hd, 2
    jg .w16_loop
    RET
.w32:
    movu                m4, [tlq+2]
    psubw               m5, m4, m3
    pabsw               m6, m5
.w32_loop:
    sub                tlq, 2
    vpbroadcastw        m2, [tlq]
    PAETH                4, 5, 6
    mova            [dstq], m0
    add               dstq, strideq
    dec                 hd
    jg .w32_loop
    RET
.w64:
    movu                m4, [tlq+ 2]
    movu                m7, [tlq+66]
    psubw               m5, m4, m3
    psubw               m8, m7, m3
    pabsw               m6, m5
    pabsw               m9, m8
.w64_loop:
    sub                tlq, 2
    vpbroadcastw        m2, [tlq]
    PAETH                4, 5, 6
    mova       [dstq+64*0], m0
    PAETH                7, 8, 9
    mova       [dstq+64*1], m0
    add               dstq, strideq
    dec                 hd
    jg .w64_loop
    RET

cglobal ipred_smooth_v_16bpc, 3, 7, 7, dst, stride, tl, w, h, weights, stride3
%define base r6-$$
    lea                  r6, [$$]
    tzcnt                wd, wm
    mov                  hd, hm
    movsxd               wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq*4]
    lea            weightsq, [base+smooth_weights_1d_16bpc+hq*4]
    neg                  hq
    vpbroadcastw         m6, [tlq+hq*2] ; bottom
    lea                  wq, [base+ipred_smooth_v_16bpc_avx512icl_table+wq]
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    vpbroadcastq         m5, [tlq+2]    ; top
    movsldup             m4, [ipred_shuf]
    psubw                m5, m6         ; top - bottom
.w4_loop:
    vbroadcasti32x4      m3, [weightsq+hq*2]
    pshufb               m3, m4
    pmulhrsw             m3, m5
    paddw                m3, m6
    vextracti32x4       xm0, m3, 3
    vextracti32x4       xm1, ym3, 1
    vextracti32x4       xm2, m3, 2
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    add                  hq, 8
    jg .end
    lea                dstq, [dstq+strideq*4]
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    jl .w4_loop
.end:
    RET
.w8:
    vbroadcasti32x4      m5, [tlq+2]    ; top
    movsldup             m4, [ipred_shuf]
    psubw                m5, m6         ; top - bottom
.w8_loop:
    vpbroadcastq         m0, [weightsq+hq*2]
    pshufb               m0, m4
    pmulhrsw             m0, m5
    paddw                m0, m6
    vextracti32x4 [dstq+strideq*0], m0, 3
    vextracti32x4 [dstq+strideq*1], ym0, 1
    vextracti32x4 [dstq+strideq*2], m0, 2
    mova          [dstq+stride3q ], xm0
    lea                dstq, [dstq+strideq*4]
    add                  hq, 4
    jl .w8_loop
    RET
.w16:
    vbroadcasti32x8      m5, [tlq+2]    ; top
    movsldup             m4, [ipred_shuf]
    psubw                m5, m6         ; top - bottom
.w16_loop:
    vpbroadcastd         m0, [weightsq+hq*2+0]
    vpbroadcastd         m1, [weightsq+hq*2+4]
    pshufb               m0, m4
    pshufb               m1, m4
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    paddw                m0, m6
    paddw                m1, m6
    vextracti32x8 [dstq+strideq*0], m0, 1
    mova          [dstq+strideq*1], ym0
    vextracti32x8 [dstq+strideq*2], m1, 1
    mova          [dstq+stride3q ], ym1
    lea                dstq, [dstq+strideq*4]
    add                  hq, 4
    jl .w16_loop
    RET
.w32:
    movu                 m5, [tlq+2]
    psubw                m5, m6
.w32_loop:
    vpbroadcastw         m0, [weightsq+hq*2+0]
    vpbroadcastw         m1, [weightsq+hq*2+2]
    vpbroadcastw         m2, [weightsq+hq*2+4]
    vpbroadcastw         m3, [weightsq+hq*2+6]
    REPX   {pmulhrsw x, m5}, m0, m1, m2, m3
    REPX   {paddw    x, m6}, m0, m1, m2, m3
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    mova   [dstq+strideq*2], m2
    mova   [dstq+stride3q ], m3
    lea                dstq, [dstq+strideq*4]
    add                  hq, 4
    jl .w32_loop
    RET
.w64:
    movu                 m4, [tlq+ 2]
    movu                 m5, [tlq+66]
    psubw                m4, m6
    psubw                m5, m6
.w64_loop:
    vpbroadcastw         m1, [weightsq+hq*2+0]
    vpbroadcastw         m3, [weightsq+hq*2+2]
    pmulhrsw             m0, m4, m1
    pmulhrsw             m1, m5
    pmulhrsw             m2, m4, m3
    pmulhrsw             m3, m5
    REPX      {paddw x, m6}, m0, m1, m2, m3
    mova [dstq+strideq*0+64*0], m0
    mova [dstq+strideq*0+64*1], m1
    mova [dstq+strideq*1+64*0], m2
    mova [dstq+strideq*1+64*1], m3
    lea                dstq, [dstq+strideq*2]
    add                  hq, 2
    jl .w64_loop
    RET

cglobal ipred_smooth_h_16bpc, 3, 7, 7, dst, stride, tl, w, h, stride3
    lea                  r6, [$$]
    mov                  wd, wm
    movifnidn            hd, hm
    vpbroadcastw         m6, [tlq+wq*2] ; right
    tzcnt                wd, wd
    add                  hd, hd
    movsxd               wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq*4]
    sub                 tlq, hq
    lea            stride3q, [strideq*3]
    lea                  wq, [base+ipred_smooth_h_16bpc_avx512icl_table+wq]
    jmp                  wq
.w4:
    movsldup             m4, [base+ipred_shuf]
    vpbroadcastq         m5, [base+smooth_weights_1d_16bpc+4*2]
.w4_loop:
    vbroadcasti32x4      m0, [tlq+hq-16] ; left
    pshufb               m0, m4
    psubw                m0, m6          ; left - right
    pmulhrsw             m0, m5
    paddw                m0, m6
    vextracti32x4       xm1, m0, 2
    vextracti32x4       xm2, ym0, 1
    vextracti32x4       xm3, m0, 3
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movq   [dstq+strideq*2], xm2
    movq   [dstq+stride3q ], xm3
    sub                  hd, 8*2
    jl .end
    lea                dstq, [dstq+strideq*4]
    movhps [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm2
    movhps [dstq+stride3q ], xm3
    lea                dstq, [dstq+strideq*4]
    jg .w4_loop
.end:
    RET
.w8:
    movsldup             m4, [base+ipred_shuf]
    vbroadcasti32x4      m5, [base+smooth_weights_1d_16bpc+8*2]
.w8_loop:
    vpbroadcastq         m0, [tlq+hq-8] ; left
    pshufb               m0, m4
    psubw                m0, m6         ; left - right
    pmulhrsw             m0, m5
    paddw                m0, m6
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], m0, 2
    vextracti32x4 [dstq+strideq*2], ym0, 1
    vextracti32x4 [dstq+stride3q ], m0, 3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4*2
    jg .w8_loop
    RET
.w16:
    movsldup             m4, [base+ipred_shuf]
    vbroadcasti32x8      m5, [base+smooth_weights_1d_16bpc+16*2]
.w16_loop:
    vpbroadcastd         m0, [tlq+hq-4]
    vpbroadcastd         m1, [tlq+hq-8]
    pshufb               m0, m4
    pshufb               m1, m4
    psubw                m0, m6
    psubw                m1, m6
    pmulhrsw             m0, m5
    pmulhrsw             m1, m5
    paddw                m0, m6
    paddw                m1, m6
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    mova          [dstq+strideq*2], ym1
    vextracti32x8 [dstq+stride3q ], m1, 1
    lea                dstq, [dstq+strideq*4]
    sub                  hq, 4*2
    jg .w16_loop
    RET
.w32:
    movu                 m5, [base+smooth_weights_1d_16bpc+32*2]
.w32_loop:
    vpbroadcastq         m3, [tlq+hq-8]
    punpcklwd            m3, m3
    psubw                m3, m6
    pshufd               m0, m3, q3333
    pshufd               m1, m3, q2222
    pshufd               m2, m3, q1111
    pshufd               m3, m3, q0000
    REPX   {pmulhrsw x, m5}, m0, m1, m2, m3
    REPX   {paddw    x, m6}, m0, m1, m2, m3
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    mova   [dstq+strideq*2], m2
    mova   [dstq+stride3q ], m3
    lea                dstq, [dstq+strideq*4]
    sub                  hq, 4*2
    jg .w32_loop
    RET
.w64:
    movu                 m4, [base+smooth_weights_1d_16bpc+64*2]
    movu                 m5, [base+smooth_weights_1d_16bpc+64*3]
.w64_loop:
    vpbroadcastw         m1, [tlq+hq-2]
    vpbroadcastw         m3, [tlq+hq-4]
    psubw                m1, m6
    psubw                m3, m6
    pmulhrsw             m0, m4, m1
    pmulhrsw             m1, m5
    pmulhrsw             m2, m4, m3
    pmulhrsw             m3, m5
    REPX      {paddw x, m6}, m0, m1, m2, m3
    mova [dstq+strideq*0+64*0], m0
    mova [dstq+strideq*0+64*1], m1
    mova [dstq+strideq*1+64*0], m2
    mova [dstq+strideq*1+64*1], m3
    lea                dstq, [dstq+strideq*2]
    sub                  hq, 2*2
    jg .w64_loop
    RET

cglobal ipred_smooth_16bpc, 3, 7, 16, dst, stride, tl, w, h, v_weights, stride3
    lea                 r6, [$$]
    mov                 wd, wm
    movifnidn           hd, hm
    vpbroadcastw       m13, [tlq+wq*2]   ; right
    tzcnt               wd, wd
    add                 hd, hd
    movsxd              wq, [base+ipred_smooth_16bpc_avx512icl_table+wq*4]
    mov                r5d, 0x55555555
    sub                tlq, hq
    mova               m14, [base+smooth_perm]
    kmovd               k1, r5d
    vpbroadcastw        m0, [tlq]        ; bottom
    mov                 r5, 0x3333333333333333
    pxor               m15, m15
    lea                 wq, [base+ipred_smooth_16bpc_avx512icl_table+wq]
    kmovq               k2, r5
    lea         v_weightsq, [base+smooth_weights_2d_16bpc+hq*2]
    jmp                 wq
.w4:
    vpbroadcastq        m5, [tlq+hq+2]
    movshdup            m3, [base+ipred_shuf]
    movsldup            m4, [base+ipred_shuf]
    vbroadcasti32x4     m6, [base+smooth_weights_2d_16bpc+4*4]
    lea           stride3q, [strideq*3]
    punpcklwd           m5, m0           ; top, bottom
.w4_loop:
    vbroadcasti32x4     m0, [v_weightsq]
    vpbroadcastq        m2, [tlq+hq-8]
    mova                m1, m13
    pshufb              m0, m3
    pmaddwd             m0, m5
    pshufb          m1{k2}, m2, m4       ; left, right
    vpdpwssd            m0, m1, m6
    vpermb              m0, m14, m0
    pavgw              ym0, ym15
    vextracti32x4      xm1, ym0, 1
    movq   [dstq+strideq*0], xm0
    movq   [dstq+strideq*1], xm1
    movhps [dstq+strideq*2], xm0
    movhps [dstq+stride3q ], xm1
    lea               dstq, [dstq+strideq*4]
    add         v_weightsq, 4*4
    sub                 hd, 4*2
    jg .w4_loop
    RET
.w8:
    vbroadcasti32x4    ym5, [tlq+hq+2]
    movshdup            m6, [base+ipred_shuf]
    movsldup            m7, [base+ipred_shuf]
    pmovzxwd            m5, ym5
    vbroadcasti32x8     m8, [base+smooth_weights_2d_16bpc+8*4]
    lea           stride3q, [strideq*3]
    vpblendmw       m5{k1}, m0, m5       ; top, bottom
.w8_loop:
    vpbroadcastq        m0, [v_weightsq+0]
    vpbroadcastq        m1, [v_weightsq+8]
    vpbroadcastd        m3, [tlq+hq-4]
    vpbroadcastd        m4, [tlq+hq-8]
    pshufb              m0, m6
    pmaddwd             m0, m5
    pshufb              m1, m6
    pmaddwd             m1, m5
    mova                m2, m13
    pshufb          m2{k2}, m3, m7       ; left, right
    mova                m3, m13
    pshufb          m3{k2}, m4, m7
    vpdpwssd            m0, m2, m8
    vpdpwssd            m1, m3, m8
    add         v_weightsq, 4*4
    vpermt2b            m0, m14, m1
    pavgw               m0, m15
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], ym0, 1
    vextracti32x4 [dstq+strideq*2], m0, 2
    vextracti32x4 [dstq+stride3q ], m0, 3
    lea               dstq, [dstq+strideq*4]
    sub                 hd, 4*2
    jg .w8_loop
    RET
.w16:
    pmovzxwd            m5, [tlq+hq+2]
    mova                m6, [base+smooth_weights_2d_16bpc+16*4]
    vpblendmw       m5{k1}, m0, m5       ; top, bottom
.w16_loop:
    vpbroadcastd        m0, [v_weightsq+0]
    vpbroadcastd        m1, [v_weightsq+4]
    pmaddwd             m0, m5
    pmaddwd             m1, m5
    mova                m2, m13
    vpbroadcastw    m2{k1}, [tlq+hq-2] ; left, right
    mova                m3, m13
    vpbroadcastw    m3{k1}, [tlq+hq-4]
    vpdpwssd            m0, m2, m6
    vpdpwssd            m1, m3, m6
    add         v_weightsq, 2*4
    vpermt2b            m0, m14, m1
    pavgw               m0, m15
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    lea               dstq, [dstq+strideq*2]
    sub                 hq, 2*2
    jg .w16_loop
    RET
.w32:
    pmovzxwd            m5, [tlq+hq+ 2]
    pmovzxwd            m6, [tlq+hq+34]
    mova                m7, [base+smooth_weights_2d_16bpc+32*4]
    mova                m8, [base+smooth_weights_2d_16bpc+32*6]
    vpblendmw       m5{k1}, m0, m5       ; top, bottom
    vpblendmw       m6{k1}, m0, m6
.w32_loop:
    vpbroadcastd        m2, [v_weightsq+0]
    vpbroadcastd        m3, [v_weightsq+4]
    pmaddwd             m0, m5, m2
    pmaddwd             m2, m6
    pmaddwd             m1, m5, m3
    pmaddwd             m3, m6
    mova                m4, m13
    vpbroadcastw    m4{k1}, [tlq+hq-2] ; left, right
    vpdpwssd            m0, m4, m7
    vpdpwssd            m2, m4, m8
    mova                m4, m13
    vpbroadcastw    m4{k1}, [tlq+hq-4]
    vpdpwssd            m1, m4, m7
    vpdpwssd            m3, m4, m8
    add         v_weightsq, 2*4
    vpermt2b            m0, m14, m2
    vpermt2b            m1, m14, m3
    pavgw               m0, m15
    pavgw               m1, m15
    mova  [dstq+strideq*0], m0
    mova  [dstq+strideq*1], m1
    lea               dstq, [dstq+strideq*2]
    sub                 hq, 2*2
    jg .w32_loop
    RET
.w64:
    pmovzxwd            m5, [tlq+hq+ 2]
    pmovzxwd            m6, [tlq+hq+34]
    pmovzxwd            m7, [tlq+hq+66]
    pmovzxwd            m8, [tlq+hq+98]
    mova                m9, [base+smooth_weights_2d_16bpc+64*4]
    vpblendmw       m5{k1}, m0, m5       ; top, bottom
    mova               m10, [base+smooth_weights_2d_16bpc+64*5]
    vpblendmw       m6{k1}, m0, m6
    mova               m11, [base+smooth_weights_2d_16bpc+64*6]
    vpblendmw       m7{k1}, m0, m7
    mova               m12, [base+smooth_weights_2d_16bpc+64*7]
    vpblendmw       m8{k1}, m0, m8
.w64_loop:
    vpbroadcastd        m3, [v_weightsq]
    mova                m4, m13
    vpbroadcastw    m4{k1}, [tlq+hq-2] ; left, right
    pmaddwd             m0, m5, m3
    pmaddwd             m2, m6, m3
    pmaddwd             m1, m7, m3
    pmaddwd             m3, m8
    vpdpwssd            m0, m4, m9
    vpdpwssd            m2, m4, m10
    vpdpwssd            m1, m4, m11
    vpdpwssd            m3, m4, m12
    add         v_weightsq, 1*4
    vpermt2b            m0, m14, m2
    vpermt2b            m1, m14, m3
    pavgw               m0, m15
    pavgw               m1, m15
    mova       [dstq+64*0], m0
    mova       [dstq+64*1], m1
    add               dstq, strideq
    sub                 hd, 1*2
    jg .w64_loop
    RET

cglobal pal_pred_16bpc, 4, 7, 4, dst, stride, pal, idx, w, h, stride3
    lea                  r6, [pal_pred_16bpc_avx512icl_table]
    tzcnt                wd, wm
    mova                 m2, [pal_pred_perm]
    movsxd               wq, [r6+wq*4]
    mova                xm3, [palq]
    movifnidn            hd, hm
    add                  wq, r6
    lea            stride3q, [strideq*3]
    jmp                  wq
.w4:
    pmovzxbw            ym0, [idxq]
    add                idxq, 16
    vpermw              ym0, ym0, ym3
    vextracti32x4       xm1, ym0, 1
    movq   [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm0
    movq   [dstq+strideq*2], xm1
    movhps [dstq+stride3q ], xm1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4
    RET
.w8:
    pmovzxbw             m0, [idxq]
    add                idxq, 32
    vpermw               m0, m0, m3
    mova          [dstq+strideq*0], xm0
    vextracti32x4 [dstq+strideq*1], ym0, 1
    vextracti32x4 [dstq+strideq*2], m0, 2
    vextracti32x4 [dstq+stride3q ], m0, 3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w8
    RET
.w16:
    vpermb               m1, m2, [idxq]
    add                idxq, 64
    vpermw               m0, m1, m3
    psrlw                m1, 8
    vpermw               m1, m1, m3
    mova          [dstq+strideq*0], ym0
    vextracti32x8 [dstq+strideq*1], m0, 1
    mova          [dstq+strideq*2], ym1
    vextracti32x8 [dstq+stride3q ], m1, 1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w16
    RET
.w32:
    vpermb               m1, m2, [idxq]
    add                idxq, 64
    vpermw               m0, m1, m3
    psrlw                m1, 8
    vpermw               m1, m1, m3
    mova   [dstq+strideq*0], m0
    mova   [dstq+strideq*1], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32
    RET
.w64:
    vpermb               m1, m2, [idxq]
    add                idxq, 64
    vpermw               m0, m1, m3
    psrlw                m1, 8
    vpermw               m1, m1, m3
    mova        [dstq+64*0], m0
    mova        [dstq+64*1], m1
    add                dstq, strideq
    dec                  hd
    jg .w64
    RET

; The ipred_filter SIMD processes 4x2 blocks in the following order which
; increases parallelism compared to doing things row by row.
;     w4     w8       w16             w32
;     1     1 2     1 2 5 6     1 2 5 6 9 a d e
;     2     2 3     2 3 6 7     2 3 6 7 a b e f
;     3     3 4     3 4 7 8     3 4 7 8 b c f g
;     4     4 5     4 5 8 9     4 5 8 9 c d g h

cglobal ipred_filter_16bpc, 4, 7, 14, dst, stride, tl, w, h, filter, top
%define base r6-$$
    lea                  r6, [$$]
%ifidn filterd, filterm
    movzx           filterd, filterb
%else
    movzx           filterd, byte filterm
%endif
    shl             filterd, 6
    movifnidn            hd, hm
    movu                xm0, [tlq-6]
    pmovsxbw             m7, [base+filter_intra_taps+filterq+32*0]
    pmovsxbw             m8, [base+filter_intra_taps+filterq+32*1]
    mov                 r5d, r8m ; bitdepth_max
    movsldup             m9, [base+filter_permA]
    movshdup            m10, [base+filter_permA]
    shr                 r5d, 11  ; is_12bpc
    jnz .12bpc
    psllw                m7, 2   ; upshift multipliers so that packusdw
    psllw                m8, 2   ; will perform clipping for free
.12bpc:
    vpbroadcastd         m5, [base+filter_rnd+r5*8]
    vpbroadcastd         m6, [base+filter_shift+r5*8]
    sub                  wd, 8
    jl .w4
.w8:
    call .main4
    movsldup            m11, [filter_permB]
    lea                 r5d, [hq*2+2]
    movshdup            m12, [filter_permB]
    lea                topq, [tlq+2]
    mova                m13, [filter_permC]
    sub                  hd, 4
    vinserti32x4        ym0, [topq], 1 ; a0 b0   t0 t1
    sub                 tlq, r5
%if WIN64
    push                 r7
    push                 r8
%endif
    mov                  r7, dstq
    mov                 r8d, hd
.w8_loop:
    movlps              xm4, xm0, [tlq+hq*2]
    call .main8
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jge .w8_loop
    test                 wd, wd
    jz .end
    mov                 r2d, 0x0d
    kmovb                k1, r2d
    lea                  r2, [strideq*3]
.w16:
    movd               xmm0, [r7+strideq*1+12]
    vpblendd           xmm0, [topq+8], 0x0e ; t1 t2
    pinsrw              xm4, xmm0, [r7+strideq*0+14], 2
    call .main8
    add                  r7, 16
    vinserti32x4        ym0, [topq+16], 1   ; a2 b2   t2 t3
    mov                  hd, r8d
    mov                dstq, r7
    add                topq, 16
.w16_loop:
    movd               xmm1, [dstq+strideq*2-4]
    punpcklwd           xm4, xmm1, xmm0
    movd               xmm0, [dstq+r2-4]
    shufps          xm4{k1}, xmm0, xm0, q3210
    call .main8
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jge .w16_loop
    sub                  wd, 8
    jg .w16
.end:
    vpermb               m2, m11, m0
    mova                ym1, ym5
    vpdpwssd             m1, m2, m7
    vpermb               m2, m12, m0
    vpdpwssd             m1, m2, m8
%if WIN64
    pop                  r8
    pop                  r7
%endif
    vextracti32x8       ym2, m1, 1
    paddd               ym1, ym2
    packusdw            ym1, ym1
    vpsrlvw             ym1, ym6
    vpermt2q             m0, m13, m1
    vextracti32x4 [dstq+strideq*0], m0, 2
    vextracti32x4 [dstq+strideq*1], ym0, 1
    RET
.w4_loop:
    movlps              xm0, [tlq-10]
    lea                dstq, [dstq+strideq*2]
    sub                 tlq, 4
.w4:
    call .main4
    movq   [dstq+strideq*0], xm0
    movhps [dstq+strideq*1], xm0
    sub                  hd, 2
    jg .w4_loop
    RET
ALIGN function_align
.main4:
    vpermb               m2, m9, m0
    mova                ym1, ym5
    vpdpwssd             m1, m2, m7
    vpermb               m0, m10, m0
    vpdpwssd             m1, m0, m8
    vextracti32x8       ym0, m1, 1
    paddd               ym0, ym1
    vextracti32x4       xm1, ym0, 1
    packusdw            xm0, xm1     ; clip
    vpsrlvw             xm0, xm6
    ret
ALIGN function_align
.main8:
    vpermb               m3, m11, m0
    mova                ym2, ym5
    vpdpwssd             m2, m3, m7
    vpermb               m3, m9, m4
    mova                ym1, ym5
    vpdpwssd             m1, m3, m7
    vpermb               m3, m12, m0
    vpdpwssd             m2, m3, m8
    vpermb               m3, m10, m4
    vpdpwssd             m1, m3, m8
    vextracti32x8       ym4, m2, 1
    vextracti32x8       ym3, m1, 1
    paddd               ym2, ym4
    paddd               ym1, ym3
    packusdw            ym1, ym2     ; clip
    vpsrlvw             ym1, ym6
    vpermt2q             m0, m13, m1 ; c0 d0   b0 b1   a0 a1
    vextracti32x4 [dstq+strideq*0], m0, 2
    vextracti32x4 [dstq+strideq*1], ym0, 1
    ret

%endif