; Copyright © 2018-2021, VideoLAN and dav1d authors
; Copyright © 2018, Two Orioles, LLC
; All rights reserved.
;
; Redistribution and use in source and binary forms, with or without
; modification, are permitted provided that the following conditions are met:
;
; 1. Redistributions of source code must retain the above copyright notice, this
;    list of conditions and the following disclaimer.
;
; 2. Redistributions in binary form must reproduce the above copyright notice,
;    this list of conditions and the following disclaimer in the documentation
;    and/or other materials provided with the distribution.
;
; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

%include "config.asm"
%include "ext/x86/x86inc.asm"

SECTION_RODATA 16

%macro SMOOTH_WEIGHT_TABLE 1-*
    %rep %0
        db %1-128, 127-%1
        %rotate 1
    %endrep
%endmacro

; sm_weights[], but modified to precalculate x and 256-x with offsets to
; enable efficient use of pmaddubsw (which requires signed values)
smooth_weights: SMOOTH_WEIGHT_TABLE         \
      0,   0, 255, 128, 255, 149,  85,  64, \
    255, 197, 146, 105,  73,  50,  37,  32, \
    255, 225, 196, 170, 145, 123, 102,  84, \
     68,  54,  43,  33,  26,  20,  17,  16, \
    255, 240, 225, 210, 196, 182, 169, 157, \
    145, 133, 122, 111, 101,  92,  83,  74, \
     66,  59,  52,  45,  39,  34,  29,  25, \
     21,  17,  14,  12,  10,   9,   8,   8, \
    255, 248, 240, 233, 225, 218, 210, 203, \
    196, 189, 182, 176, 169, 163, 156, 150, \
    144, 138, 133, 127, 121, 116, 111, 106, \
    101,  96,  91,  86,  82,  77,  73,  69, \
     65,  61,  57,  54,  50,  47,  44,  41, \
     38,  35,  32,  29,  27,  25,  22,  20, \
     18,  16,  15,  13,  12,  10,   9,   8, \
      7,   6,   6,   5,   5,   4,   4,   4

ipred_v_shuf:     db  0,  1,  0,  1,  2,  3,  2,  3,  4,  5,  4,  5,  6,  7,  6,  7
ipred_h_shuf:     db  3,  3,  3,  3,  2,  2,  2,  2,  1,  1,  1,  1,  0,  0,  0,  0
ipred_paeth_shuf: db  1,  1,  1,  1,  1,  1,  1,  1,  0,  0,  0,  0,  0,  0,  0,  0
filter_shuf1:     db  3,  4,  3,  4,  5,  6,  5,  6,  7,  2,  7,  2,  1, -1,  1, -1
filter_shuf2:     db  3,  4,  3,  4,  5,  6,  5,  6,  7, 11,  7, 11, 15, -1, 15, -1
z_filter_wh4:     db  7,  7, 19,  7,
z_filter_wh8:     db 19, 19, 11, 19, 11, 15, 15, 15, 23, 23, 23, 23, 39, 39, 39, 39
pb_8:             times 8 db 8
pd_32768:         dd 32768
z_filter_wh16:    db 19, 19, 19, 23, 23, 23, 31, 31, 31, 47, 47, 47, 79, 79, 79, -1
z_filter_t_w48:   db 55,127,  7,127, 15, 31, 39, 31,127, 39,127, 39,  7, 15, 31, 15
                  db 39, 63,  3, 63,  3,  3, 19,  3, 47, 19, 47, 19,  3,  3,  3,  3
z_filter_t_w16:   db 15, 31,  7, 15, 31,  7,  3, 31,  3,  3,  3,  3,  3,  3,  0,  0
z_upsample1:      db  1,  0,  2,  1,  3,  2,  4,  3,  5,  4,  6,  5,  7,  6,  8,  7
z_upsample2:      db  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,  8,  8,  8,  8,  8
z1_shuf_w4:       db  0,  1,  1,  2,  2,  3,  3,  4,  8,  9,  9, 10, 10, 11, 11, 12
pb_0to15:         db  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15
z_base_inc:       dw   0*64,   1*64,   2*64,   3*64,   4*64,   5*64,   6*64,   7*64
z_filter_s:       db  0,  0,  0,  1,  1,  2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7
                  db  7,  8,  8,  9,  9, 10, 10, 11
z_filter_k_tail:  db  0, 64,  0, 64,  8, 56,  0, 64
z_filter_k:       times  4 db  0, 16
                  times  4 db  0, 20
                  times  4 db  8, 16
                  times  4 db 32, 16
                  times  4 db 24, 20
                  times  4 db 16, 16
                  times  4 db  0,  0
                  times  4 db  0,  0
pw_8:             times  8 db  8,  0
pb_3:             times 16 db 3
pw_62:            times  8 dw 62
pw_64:            times  8 dw 64
pw_256:           times  8 dw 256
pw_512:           times  8 dw 512
pw_m256:          times  8 dw -256
pb_2:             times  8 db 2
pb_4:             times  8 db 4
pb_128:           times  8 db 128
pb_m16:           times  8 db -16
pw_128:           times  4 dw 128
pw_255:           times  4 dw 255
pb_36_m4:         times  4 db 36, -4
pb_127_m127:      times  4 db 127, -127

%macro JMP_TABLE 3-*
    %xdefine %1_%2_table (%%table - 2*4)
    %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2)
    %%table:
    %rep %0 - 2
        dd %%base %+ .%3 - (%%table - 2*4)
        %rotate 1
    %endrep
%endmacro

%define ipred_dc_splat_ssse3_table (ipred_dc_ssse3_table + 10*4)
%define ipred_cfl_splat_ssse3_table (ipred_cfl_ssse3_table + 8*4)

JMP_TABLE ipred_h,          ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_dc,         ssse3, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \
                                s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4
JMP_TABLE ipred_dc_left,    ssse3, h4, h8, h16, h32, h64
JMP_TABLE ipred_smooth,     ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_smooth_v,   ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_smooth_h,   ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_paeth,      ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_z1,         ssse3, w4, w8, w16, w32, w64
JMP_TABLE pal_pred,         ssse3, w4, w8, w16, w32, w64
JMP_TABLE ipred_cfl,        ssse3, h4, h8, h16, h32, w4, w8, w16, w32, \
                                s4-8*4, s8-8*4, s16-8*4, s32-8*4
JMP_TABLE ipred_cfl_left,   ssse3, h4, h8, h16, h32
JMP_TABLE ipred_filter,     ssse3, w4, w8, w16, w32

cextern dr_intra_derivative
cextern filter_intra_taps

SECTION .text

;---------------------------------------------------------------------------------------
;int dav1d_ipred_h_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
%macro IPRED_SET   3                                          ; width, stride, stride size pshuflw_imm8
    pshuflw                      m1, m0, %3                   ; extend 8 byte for 2 pos
    punpcklqdq                   m1, m1
    mova           [dstq +      %2], m1
%if %1 > 16
    mova           [dstq + 16 + %2], m1
%endif
%if %1 > 32
    mova           [dstq + 32 + %2], m1
    mova           [dstq + 48 + %2], m1
%endif
%endmacro

%macro IPRED_H 1                                            ; width
    sub                         tlq, 4
    movd                         m0, [tlq]                  ; get 4 bytes of topleft data
    punpcklbw                    m0, m0                     ; extend 2 byte
%if %1 == 4
    pshuflw                      m1, m0, q2233
    movd           [dstq+strideq*0], m1
    psrlq                        m1, 32
    movd           [dstq+strideq*1], m1
    pshuflw                      m0, m0, q0011
    movd           [dstq+strideq*2], m0
    psrlq                        m0, 32
    movd           [dstq+stride3q ], m0

%elif %1 == 8
    punpcklwd                    m0, m0
    punpckhdq                    m1, m0, m0
    punpckldq                    m0, m0
    movq           [dstq+strideq*1], m1
    movhps         [dstq+strideq*0], m1
    movq           [dstq+stride3q ], m0
    movhps         [dstq+strideq*2], m0
%else
    IPRED_SET                    %1,         0, q3333
    IPRED_SET                    %1,   strideq, q2222
    IPRED_SET                    %1, strideq*2, q1111
    IPRED_SET                    %1,  stride3q, q0000
%endif
    lea                        dstq, [dstq+strideq*4]
    sub                          hd, 4
    jg .w%1
    RET
%endmacro

INIT_XMM ssse3
cglobal ipred_h_8bpc, 3, 6, 2, dst, stride, tl, w, h, stride3
    LEA                          r5, ipred_h_ssse3_table
    tzcnt                        wd, wm
    movifnidn                    hd, hm
    movsxd                       wq, [r5+wq*4]
    add                          wq, r5
    lea                    stride3q, [strideq*3]
    jmp                          wq
.w4:
    IPRED_H                       4
.w8:
    IPRED_H                       8
.w16:
    IPRED_H                      16
.w32:
    IPRED_H                      32
.w64:
    IPRED_H                      64

;---------------------------------------------------------------------------------------
;int dav1d_ipred_v_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_v_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3
    LEA                  r5, ipred_dc_splat_ssse3_table
    tzcnt                wd, wm
    movu                 m0, [tlq+ 1]
    movu                 m1, [tlq+17]
    movu                 m2, [tlq+33]
    movu                 m3, [tlq+49]
    movifnidn            hd, hm
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  wq

;---------------------------------------------------------------------------------------
;int dav1d_ipred_dc_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_dc_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3
    movifnidn                    hd, hm
    movifnidn                    wd, wm
    tzcnt                       r6d, hd
    lea                         r5d, [wq+hq]
    movd                         m4, r5d
    tzcnt                       r5d, r5d
    movd                         m5, r5d
    LEA                          r5, ipred_dc_ssse3_table
    tzcnt                        wd, wd
    movsxd                       r6, [r5+r6*4]
    movsxd                       wq, [r5+wq*4+20]
    pcmpeqd                      m3, m3
    psrlw                        m4, 1                             ; dc = (width + height) >> 1;
    add                          r6, r5
    add                          wq, r5
    lea                    stride3q, [strideq*3]
    jmp r6
.h4:
    movd                         m0, [tlq-4]
    pmaddubsw                    m0, m3
    jmp                          wq
.w4:
    movd                         m1, [tlq+1]
    pmaddubsw                    m1, m3
    psubw                        m0, m4
    paddw                        m0, m1
    pmaddwd                      m0, m3
    cmp                          hd, 4
    jg .w4_mul
    psrlw                        m0, 3                             ; dc >>= ctz(width + height);
    jmp .w4_end
.w4_mul:
    punpckhqdq                   m1, m0, m0
    paddw                        m0, m1
    psrlq                        m1, m0, 32
    paddw                        m0, m1
    psrlw                        m0, 2
    mov                         r6d, 0x5556
    mov                         r2d, 0x3334
    test                         hd, 8
    cmovz                       r6d, r2d
    movd                         m5, r6d
    pmulhuw                      m0, m5
.w4_end:
    pxor                         m1, m1
    pshufb                       m0, m1
.s4:
    movd           [dstq+strideq*0], m0
    movd           [dstq+strideq*1], m0
    movd           [dstq+strideq*2], m0
    movd           [dstq+stride3q ], m0
    lea                        dstq, [dstq+strideq*4]
    sub                          hd, 4
    jg .s4
    RET
ALIGN function_align
.h8:
    movq                         m0, [tlq-8]
    pmaddubsw                    m0, m3
    jmp                          wq
.w8:
    movq                         m1, [tlq+1]
    pmaddubsw                    m1, m3
    psubw                        m4, m0
    punpckhqdq                   m0, m0
    psubw                        m0, m4
    paddw                        m0, m1
    pshuflw                      m1, m0, q1032                  ; psrlq  m1, m0, 32
    paddw                        m0, m1
    pmaddwd                      m0, m3
    psrlw                        m0, m5
    cmp                          hd, 8
    je .w8_end
    mov                         r6d, 0x5556
    mov                         r2d, 0x3334
    cmp                          hd, 32
    cmovz                       r6d, r2d
    movd                         m1, r6d
    pmulhuw                      m0, m1
.w8_end:
    pxor                         m1, m1
    pshufb                       m0, m1
.s8:
    movq           [dstq+strideq*0], m0
    movq           [dstq+strideq*1], m0
    movq           [dstq+strideq*2], m0
    movq           [dstq+stride3q ], m0
    lea                        dstq, [dstq+strideq*4]
    sub                          hd, 4
    jg .s8
    RET
ALIGN function_align
.h16:
    mova                         m0, [tlq-16]
    pmaddubsw                    m0, m3
    jmp                          wq
.w16:
    movu                         m1, [tlq+1]
    pmaddubsw                    m1, m3
    paddw                        m0, m1
    psubw                        m4, m0
    punpckhqdq                   m0, m0
    psubw                        m0, m4
    pshuflw                      m1, m0, q1032                  ; psrlq  m1, m0, 32
    paddw                        m0, m1
    pmaddwd                      m0, m3
    psrlw                        m0, m5
    cmp                          hd, 16
    je .w16_end
    mov                         r6d, 0x5556
    mov                         r2d, 0x3334
    test                         hd, 8|32
    cmovz                       r6d, r2d
    movd                         m1, r6d
    pmulhuw                      m0, m1
.w16_end:
    pxor                         m1, m1
    pshufb                       m0, m1
.s16:
    mova           [dstq+strideq*0], m0
    mova           [dstq+strideq*1], m0
    mova           [dstq+strideq*2], m0
    mova           [dstq+stride3q ], m0
    lea                        dstq, [dstq+strideq*4]
    sub                          hd, 4
    jg .s16
    RET
ALIGN function_align
.h32:
    mova                         m0, [tlq-32]
    pmaddubsw                    m0, m3
    mova                         m2, [tlq-16]
    pmaddubsw                    m2, m3
    paddw                        m0, m2
    jmp wq
.w32:
    movu                         m1, [tlq+1]
    pmaddubsw                    m1, m3
    movu                         m2, [tlq+17]
    pmaddubsw                    m2, m3
    paddw                        m1, m2
    paddw                        m0, m1
    psubw                        m4, m0
    punpckhqdq                   m0, m0
    psubw                        m0, m4
    pshuflw                      m1, m0, q1032                   ; psrlq  m1, m0, 32
    paddw                        m0, m1
    pmaddwd                      m0, m3
    psrlw                        m0, m5
    cmp                          hd, 32
    je .w32_end
    lea                         r2d, [hq*2]
    mov                         r6d, 0x5556
    mov                         r2d, 0x3334
    test                         hd, 64|16
    cmovz                       r6d, r2d
    movd                         m1, r6d
    pmulhuw                      m0, m1
.w32_end:
    pxor                         m1, m1
    pshufb                       m0, m1
    mova                         m1, m0
.s32:
    mova                     [dstq], m0
    mova                  [dstq+16], m1
    mova             [dstq+strideq], m0
    mova          [dstq+strideq+16], m1
    mova           [dstq+strideq*2], m0
    mova        [dstq+strideq*2+16], m1
    mova            [dstq+stride3q], m0
    mova         [dstq+stride3q+16], m1
    lea                        dstq, [dstq+strideq*4]
    sub                          hd, 4
    jg .s32
    RET
ALIGN function_align
.h64:
    mova                         m0, [tlq-64]
    mova                         m1, [tlq-48]
    pmaddubsw                    m0, m3
    pmaddubsw                    m1, m3
    paddw                        m0, m1
    mova                         m1, [tlq-32]
    pmaddubsw                    m1, m3
    paddw                        m0, m1
    mova                         m1, [tlq-16]
    pmaddubsw                    m1, m3
    paddw                        m0, m1
    jmp wq
.w64:
    movu                         m1, [tlq+ 1]
    movu                         m2, [tlq+17]
    pmaddubsw                    m1, m3
    pmaddubsw                    m2, m3
    paddw                        m1, m2
    movu                         m2, [tlq+33]
    pmaddubsw                    m2, m3
    paddw                        m1, m2
    movu                         m2, [tlq+49]
    pmaddubsw                    m2, m3
    paddw                        m1, m2
    paddw                        m0, m1
    psubw                        m4, m0
    punpckhqdq                   m0, m0
    psubw                        m0, m4
    pshuflw                      m1, m0, q1032                   ; psrlq  m1, m0, 32
    paddw                        m0, m1
    pmaddwd                      m0, m3
    psrlw                        m0, m5
    cmp                          hd, 64
    je .w64_end
    mov                         r6d, 0x5556
    mov                         r2d, 0x3334
    test                         hd, 32
    cmovz                       r6d, r2d
    movd                         m1, r6d
    pmulhuw                      m0, m1
.w64_end:
    pxor                         m1, m1
    pshufb                       m0, m1
    mova                         m1, m0
    mova                         m2, m0
    mova                         m3, m0
.s64:
    mova                     [dstq], m0
    mova                  [dstq+16], m1
    mova                  [dstq+32], m2
    mova                  [dstq+48], m3
    mova             [dstq+strideq], m0
    mova          [dstq+strideq+16], m1
    mova          [dstq+strideq+32], m2
    mova          [dstq+strideq+48], m3
    lea                        dstq, [dstq+strideq*2]
    sub                          hd, 2
    jg .s64
    RET

;---------------------------------------------------------------------------------------
;int dav1d_ipred_dc_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_dc_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3
    LEA                  r5, ipred_dc_left_ssse3_table
    mov                  hd, hm                ; zero upper half
    tzcnt               r6d, hd
    sub                 tlq, hq
    tzcnt                wd, wm
    movu                 m0, [tlq]
    movd                 m3, [r5-ipred_dc_left_ssse3_table+pd_32768]
    movd                 m2, r6d
    psrld                m3, m2
    movsxd               r6, [r5+r6*4]
    pcmpeqd              m2, m2
    pmaddubsw            m0, m2
    add                  r6, r5
    add                  r5, ipred_dc_splat_ssse3_table-ipred_dc_left_ssse3_table
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    jmp                  r6
.h64:
    movu                 m1, [tlq+48]                           ; unaligned when jumping here from dc_top
    pmaddubsw            m1, m2
    paddw                m0, m1
    movu                 m1, [tlq+32]                           ; unaligned when jumping here from dc_top
    pmaddubsw            m1, m2
    paddw                m0, m1
.h32:
    movu                 m1, [tlq+16]                           ; unaligned when jumping here from dc_top
    pmaddubsw            m1, m2
    paddw                m0, m1
.h16:
    pshufd               m1, m0, q3232                          ; psrlq               m1, m0, 16
    paddw                m0, m1
.h8:
    pshuflw              m1, m0, q1032                          ; psrlq               m1, m0, 32
    paddw                m0, m1
.h4:
    pmaddwd              m0, m2
    pmulhrsw             m0, m3
    lea            stride3q, [strideq*3]
    pxor                 m1, m1
    pshufb               m0, m1
    mova                 m1, m0
    mova                 m2, m0
    mova                 m3, m0
    jmp                  wq

;---------------------------------------------------------------------------------------
;int dav1d_ipred_dc_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_dc_128_8bpc, 2, 7, 6, dst, stride, tl, w, h, stride3
    LEA                  r5, ipred_dc_splat_ssse3_table
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, [r5+wq*4]
    movddup              m0, [r5-ipred_dc_splat_ssse3_table+pb_128]
    mova                 m1, m0
    mova                 m2, m0
    mova                 m3, m0
    add                  wq, r5
    lea            stride3q, [strideq*3]
    jmp                  wq

;---------------------------------------------------------------------------------------
;int dav1d_ipred_dc_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_dc_top_8bpc, 3, 7, 6, dst, stride, tl, w, h
    LEA                  r5, ipred_dc_left_ssse3_table
    tzcnt                wd, wm
    inc                 tlq
    movu                 m0, [tlq]
    movifnidn            hd, hm
    movd                 m3, [r5-ipred_dc_left_ssse3_table+pd_32768]
    movd                 m2, wd
    psrld                m3, m2
    movsxd               r6, [r5+wq*4]
    pcmpeqd              m2, m2
    pmaddubsw            m0, m2
    add                  r6, r5
    add                  r5, ipred_dc_splat_ssse3_table-ipred_dc_left_ssse3_table
    movsxd               wq, [r5+wq*4]
    add                  wq, r5
    jmp                  r6

;---------------------------------------------------------------------------------------
;int dav1d_ipred_smooth_v_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
%macro SMOOTH 6 ; src[1-2], mul[1-2], add[1-2]
                ;            w * a         = (w - 128) * a + 128 * a
                ;            (256 - w) * b = (127 - w) * b + 129 * b
                ; => w * a + (256 - w) * b = [(w - 128) * a + (127 - w) * b] + [128 * a + 129 * b]
    pmaddubsw            m6, m%3, m%1
    pmaddubsw            m0, m%4, m%2                    ; (w - 128) * a + (127 - w) * b
    paddw                m6, m%5
    paddw                m0, m%6                         ; [(w - 128) * a + (127 - w) * b] + [128 * a + 129 * b + 128]
    psrlw                m6, 8
    psrlw                m0, 8
    packuswb             m6, m0
%endmacro

cglobal ipred_smooth_v_8bpc, 3, 7, 7, dst, stride, tl, w, h, weights
%define base r6-ipred_smooth_v_ssse3_table
    LEA                  r6, ipred_smooth_v_ssse3_table
    tzcnt                wd, wm
    mov                  hd, hm
    movsxd               wq, [r6+wq*4]
    movddup              m0, [base+pb_127_m127]
    movddup              m1, [base+pw_128]
    lea            weightsq, [base+smooth_weights+hq*4]
    neg                  hq
    movd                 m5, [tlq+hq]
    pxor                 m2, m2
    pshufb               m5, m2
    add                  wq, r6
    jmp                  wq
.w4:
    movd                 m2, [tlq+1]
    punpckldq            m2, m2
    punpcklbw            m2, m5                          ; top, bottom
    lea                  r3, [strideq*3]
    mova                 m4, [base+ipred_v_shuf]
    mova                 m5, m4
    punpckldq            m4, m4
    punpckhdq            m5, m5
    pmaddubsw            m3, m2, m0                      ; m3: 127 * top - 127 * bottom
    paddw                m1, m2                          ; m1:   1 * top + 256 * bottom + 128, overflow is ok
    paddw                m3, m1                          ; m3: 128 * top + 129 * bottom + 128
.w4_loop:
    movu                 m1, [weightsq+hq*2]
    pshufb               m0, m1, m4                      ;m2, m3, m4 and m5 should be stable in loop
    pshufb               m1, m5
    SMOOTH                0, 1, 2, 2, 3, 3
    movd   [dstq+strideq*0], m6
    pshuflw              m1, m6, q1032
    movd   [dstq+strideq*1], m1
    punpckhqdq           m6, m6
    movd   [dstq+strideq*2], m6
    psrlq                m6, 32
    movd   [dstq+r3       ], m6
    lea                dstq, [dstq+strideq*4]
    add                  hq, 4
    jl .w4_loop
    RET
ALIGN function_align
.w8:
    movq                 m2, [tlq+1]
    punpcklbw            m2, m5
    mova                 m5, [base+ipred_v_shuf]
    lea                  r3, [strideq*3]
    pshufd               m4, m5, q0000
    pshufd               m5, m5, q1111
    pmaddubsw            m3, m2, m0
    paddw                m1, m2
    paddw                m3, m1                           ; m3 is output for loop
.w8_loop:
    movq                 m1, [weightsq+hq*2]
    pshufb               m0, m1, m4
    pshufb               m1, m5
    SMOOTH                0, 1, 2, 2, 3, 3
    movq   [dstq+strideq*0], m6
    movhps [dstq+strideq*1], m6
    lea                dstq, [dstq+strideq*2]
    add                  hq, 2
    jl .w8_loop
    RET
ALIGN function_align
.w16:
    movu                 m3, [tlq+1]
    punpcklbw            m2, m3, m5
    punpckhbw            m3, m5
    pmaddubsw            m4, m2, m0
    pmaddubsw            m5, m3, m0
    paddw                m0, m1, m2
    paddw                m1, m3
    paddw                m4, m0
    paddw                m5, m1                           ; m4 and m5 is output for loop
.w16_loop:
    movd                 m1, [weightsq+hq*2]
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    SMOOTH 1, 1, 2, 3, 4, 5
    mova             [dstq], m6
    add                dstq, strideq
    add                  hq, 1
    jl .w16_loop
    RET
ALIGN function_align
.w32:
%if WIN64
    movaps         [rsp+24], xmm7
    %define xmm_regs_used 8
%endif
    mova                 m7, m5
.w32_loop_init:
    mov                 r3d, 2
.w32_loop:
    movddup              m0, [base+pb_127_m127]
    movddup              m1, [base+pw_128]
    movu                 m3, [tlq+1]
    punpcklbw            m2, m3, m7
    punpckhbw            m3, m7
    pmaddubsw            m4, m2, m0
    pmaddubsw            m5, m3, m0
    paddw                m0, m1, m2
    paddw                m1, m3
    paddw                m4, m0
    paddw                m5, m1
    movd                 m1, [weightsq+hq*2]
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    SMOOTH                1, 1, 2, 3, 4, 5
    mova             [dstq], m6
    add                 tlq, 16
    add                dstq, 16
    dec                 r3d
    jg .w32_loop
    lea                dstq, [dstq-32+strideq]
    sub                 tlq, 32
    add                  hq, 1
    jl .w32_loop_init
    RET
ALIGN function_align
.w64:
%if WIN64
    movaps         [rsp+24], xmm7
    %define xmm_regs_used 8
%endif
    mova                 m7, m5
.w64_loop_init:
    mov                 r3d, 4
.w64_loop:
    movddup              m0, [base+pb_127_m127]
    movddup              m1, [base+pw_128]
    movu                 m3, [tlq+1]
    punpcklbw            m2, m3, m7
    punpckhbw            m3, m7
    pmaddubsw            m4, m2, m0
    pmaddubsw            m5, m3, m0
    paddw                m0, m1, m2
    paddw                m1, m3
    paddw                m4, m0
    paddw                m5, m1
    movd                 m1, [weightsq+hq*2]
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    SMOOTH                1, 1, 2, 3, 4, 5
    mova             [dstq], m6
    add                 tlq, 16
    add                dstq, 16
    dec                 r3d
    jg .w64_loop
    lea                dstq, [dstq-64+strideq]
    sub                 tlq, 64
    add                  hq, 1
    jl .w64_loop_init
    RET

;---------------------------------------------------------------------------------------
;int dav1d_ipred_smooth_h_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
cglobal ipred_smooth_h_8bpc, 3, 7, 8, dst, stride, tl, w, h
%define base r6-ipred_smooth_h_ssse3_table
    LEA                  r6, ipred_smooth_h_ssse3_table
    mov                  wd, wm
    movd                 m3, [tlq+wq]
    pxor                 m1, m1
    pshufb               m3, m1                          ; right
    tzcnt                wd, wd
    mov                  hd, hm
    movsxd               wq, [r6+wq*4]
    movddup              m4, [base+pb_127_m127]
    movddup              m5, [base+pw_128]
    add                  wq, r6
    jmp                  wq
.w4:
    movddup              m6, [base+smooth_weights+4*2]
    mova                 m7, [base+ipred_h_shuf]
    sub                 tlq, 4
    sub                 tlq, hq
    lea                  r3, [strideq*3]
.w4_loop:
    movd                 m2, [tlq+hq]                    ; left
    pshufb               m2, m7
    punpcklbw            m1, m2, m3                      ; left, right
    punpckhbw            m2, m3
    pmaddubsw            m0, m1, m4                      ; 127 * left - 127 * right
    paddw                m0, m1                          ; 128 * left + 129 * right
    pmaddubsw            m1, m6
    paddw                m1, m5
    paddw                m0, m1
    pmaddubsw            m1, m2, m4
    paddw                m1, m2
    pmaddubsw            m2, m6
    paddw                m2, m5
    paddw                m1, m2
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
    movd   [dstq+strideq*0], m0
    pshuflw              m1, m0, q1032
    movd   [dstq+strideq*1], m1
    punpckhqdq           m0, m0
    movd   [dstq+strideq*2], m0
    psrlq                m0, 32
    movd   [dstq+r3       ], m0
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4_loop
    RET
ALIGN function_align
.w8:
    mova                 m6, [base+smooth_weights+8*2]
    mova                 m7, [base+ipred_h_shuf]
    sub                 tlq, 4
    sub                 tlq, hq
    punpckldq            m7, m7
.w8_loop:
    movd                 m2, [tlq+hq]                    ; left
    pshufb               m2, m7
    punpcklbw            m1, m2, m3                      ; left, right
    punpckhbw            m2, m3
    pmaddubsw            m0, m1, m4                      ; 127 * left - 127 * right
    paddw                m0, m1                          ; 128 * left + 129 * right
    pmaddubsw            m1, m6
    paddw                m1, m5
    paddw                m0, m1
    pmaddubsw            m1, m2, m4
    paddw                m1, m2
    pmaddubsw            m2, m6
    paddw                m2, m5
    paddw                m1, m2
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w8_loop
    RET
ALIGN function_align
.w16:
    mova                 m6, [base+smooth_weights+16*2]
    mova                 m7, [base+smooth_weights+16*3]
    sub                 tlq, 1
    sub                 tlq, hq
.w16_loop:
    pxor                 m1, m1
    movd                 m2, [tlq+hq]                    ; left
    pshufb               m2, m1
    punpcklbw            m1, m2, m3                      ; left, right
    punpckhbw            m2, m3
    pmaddubsw            m0, m1, m4                      ; 127 * left - 127 * right
    paddw                m0, m1                          ; 128 * left + 129 * right
    pmaddubsw            m1, m6
    paddw                m1, m5
    paddw                m0, m1
    pmaddubsw            m1, m2, m4
    paddw                m1, m2
    pmaddubsw            m2, m7
    paddw                m2, m5
    paddw                m1, m2
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
    mova             [dstq], m0
    lea                dstq, [dstq+strideq]
    sub                  hd, 1
    jg .w16_loop
    RET
ALIGN function_align
.w32:
    sub                 tlq, 1
    sub                 tlq, hq
    pxor                 m6, m6
.w32_loop_init:
    mov                  r5, 2
    lea                  r3, [base+smooth_weights+16*4]
.w32_loop:
    mova                 m7, [r3]
    add                  r3, 16
    movd                 m2, [tlq+hq]                    ; left
    pshufb               m2, m6
    punpcklbw            m1, m2, m3                      ; left, right
    punpckhbw            m2, m3
    pmaddubsw            m0, m1, m4                      ; 127 * left - 127 * right
    paddw                m0, m1                          ; 128 * left + 129 * right
    pmaddubsw            m1, m7
    paddw                m1, m5
    paddw                m0, m1
    pmaddubsw            m1, m2, m4
    paddw                m1, m2
    mova                 m7, [r3]
    add                  r3, 16
    pmaddubsw            m2, m7
    paddw                m2, m5
    paddw                m1, m2
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
    mova             [dstq], m0
    add                dstq, 16
    dec                  r5
    jg .w32_loop
    lea                dstq, [dstq-32+strideq]
    sub                  hd, 1
    jg .w32_loop_init
    RET
ALIGN function_align
.w64:
    sub                 tlq, 1
    sub                 tlq, hq
    pxor                 m6, m6
.w64_loop_init:
    mov                  r5, 4
    lea                  r3, [base+smooth_weights+16*8]
.w64_loop:
    mova                 m7, [r3]
    add                  r3, 16
    movd                 m2, [tlq+hq]                    ; left
    pshufb               m2, m6
    punpcklbw            m1, m2, m3                      ; left, right
    punpckhbw            m2, m3
    pmaddubsw            m0, m1, m4                      ; 127 * left - 127 * right
    paddw                m0, m1                          ; 128 * left + 129 * right
    pmaddubsw            m1, m7
    paddw                m1, m5
    paddw                m0, m1
    pmaddubsw            m1, m2, m4
    paddw                m1, m2
    mova                 m7, [r3]
    add                  r3, 16
    pmaddubsw            m2, m7
    paddw                m2, m5
    paddw                m1, m2
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
    mova             [dstq], m0
    add                dstq, 16
    dec                  r5
    jg .w64_loop
    lea                dstq, [dstq-64+strideq]
    sub                  hd, 1
    jg .w64_loop_init
    RET

;---------------------------------------------------------------------------------------
;int dav1d_ipred_smooth_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                                    const int width, const int height, const int a);
;---------------------------------------------------------------------------------------
%macro SMOOTH_2D_END  7                                  ; src[1-2], mul[1-2], add[1-2], m3
    pmaddubsw            m6, m%3, m%1
    mova                 m0, m6
    pmaddubsw            m6, m%4, m%2
    mova                 m1, m6
%ifnum %5
    paddw                m0, m%5
%else
    paddw                m0, %5
%endif
%ifnum %6
    paddw                m1, m%6
%else
    paddw                m1, %6
%endif
%ifnum %7
%else
    mova                 m3, %7
%endif
    pavgw                m0, m2
    pavgw                m1, m3
    psrlw                m0, 8
    psrlw                m1, 8
    packuswb             m0, m1
%endmacro

%macro SMOOTH_OUTPUT_16B  12      ; m1, [buffer1, buffer2, buffer3, buffer4,] [w1, w2,] m3, m7, [m0, m4, m5]
    mova                 m1, [rsp+16*%1]                  ; top
    punpckhbw            m6, m1, m0                       ; top, bottom
    punpcklbw            m1, m0                           ; top, bottom
    pmaddubsw            m2, m1, m5
    mova        [rsp+16*%2], m1
    paddw                m1, m3                           ;   1 * top + 255 * bottom + 255
    paddw                m2, m1                           ; 128 * top + 129 * bottom + 255
    mova        [rsp+16*%3], m2
    pmaddubsw            m2, m6, m5
    mova        [rsp+16*%4], m6
    paddw                m6, m3                           ;   1 * top + 255 * bottom + 255
    paddw                m2, m6                           ; 128 * top + 129 * bottom + 255
    mova        [rsp+16*%5], m2
    movd                 m1, [tlq+hq]                     ; left
    pshufb               m1, [base+pb_3]                  ; topleft[-(1 + y)]
    punpcklbw            m1, m4                           ; left, right
    pmaddubsw            m2, m1, m5                       ; 127 * left - 127 * right
    paddw                m2, m1                           ; 128 * left + 129 * right
    mova                 m3, m2
    pmaddubsw            m0, m1, %6                       ; weights_hor = &dav1d_sm_weights[width];
    pmaddubsw            m1, %7
    paddw                m2, m3, m0
    paddw                m3, m1
    movd                 m1, [v_weightsq]                 ; weights_ver = &dav1d_sm_weights[height];
    mova                 m7, [rsp+16*%9]
    pshufb               m1, m7
    mova        [rsp+16*%8], m3
    mova                 m4, [rsp+16*%2]
    mova                 m5, [rsp+16*%3]
    mova                 m3, [rsp+16*%4]
    mova                 m7, [rsp+16*%5]
    SMOOTH_2D_END         1, 1, 4, 3, 5, 7, [rsp+16*%8]
    mova             [dstq], m0
    movddup              m3, [base+pw_255]                ; recovery
    mova                 m0, [rsp+16*%10]                 ; recovery
    mova                 m4, [rsp+16*%11]                 ; recovery
    mova                 m5, [rsp+16*%12]                 ; recovery
%endmacro

cglobal ipred_smooth_8bpc, 3, 7, 8, -13*16, dst, stride, tl, w, h, v_weights
%define base r6-ipred_smooth_ssse3_table
    mov                  wd, wm
    mov                  hd, hm
    LEA                  r6, ipred_smooth_ssse3_table
    movd                 m4, [tlq+wq]                     ; right
    pxor                 m2, m2
    pshufb               m4, m2
    tzcnt                wd, wd
    mov                  r5, tlq
    sub                  r5, hq
    movsxd               wq, [r6+wq*4]
    movddup              m5, [base+pb_127_m127]
    movd                 m0, [r5]
    pshufb               m0, m2                           ; bottom
    movddup              m3, [base+pw_255]
    add                  wq, r6
    lea          v_weightsq, [base+smooth_weights+hq*2]   ; weights_ver = &dav1d_sm_weights[height]
    jmp                  wq
.w4:
    mova                 m7, [base+ipred_v_shuf]
    movd                 m1, [tlq+1]                      ; left
    pshufd               m1, m1, q0000
    sub                 tlq, 4
    lea                  r3, [strideq*3]
    sub                 tlq, hq
    punpcklbw            m1, m0                           ; top, bottom
    pshufd               m6, m7, q1100
    pshufd               m7, m7, q3322
    pmaddubsw            m2, m1, m5
    paddw                m3, m1                           ;   1 * top + 255 * bottom + 255
    paddw                m2, m3                           ; 128 * top + 129 * bottom + 255
    mova         [rsp+16*0], m1
    mova         [rsp+16*1], m2
    movq                 m1,  [base+smooth_weights+4*2]   ; weights_hor = &dav1d_sm_weights[width];
    punpcklqdq           m1, m1
    mova         [rsp+16*2], m1
    mova         [rsp+16*3], m4
    mova         [rsp+16*4], m6
    mova         [rsp+16*5], m5
.w4_loop:
    movd                 m1, [tlq+hq]                 ; left
    pshufb               m1, [base+ipred_h_shuf]
    punpcklbw            m0, m1, m4                   ; left, right
    punpckhbw            m1, m4
    pmaddubsw            m2, m0, m5                   ; 127 * left - 127 * right
    pmaddubsw            m3, m1, m5
    paddw                m2, m0                       ; 128 * left + 129 * right
    paddw                m3, m1
    mova                 m4, [rsp+16*2]
    pmaddubsw            m0, m4
    pmaddubsw            m1, m4
    paddw                m2, m0
    paddw                m3, m1
    movq                 m1, [v_weightsq]             ; weights_ver = &dav1d_sm_weights[height];
    add          v_weightsq, 8
    pshufb               m0, m1, m6
    pshufb               m1, m7
    mova                 m4, [rsp+16*0]
    mova                 m5, [rsp+16*1]
    SMOOTH_2D_END         0, 1, 4, 4, 5, 5, 3
    mova                 m4, [rsp+16*3]
    mova                 m6, [rsp+16*4]
    mova                 m5, [rsp+16*5]
    movd   [dstq+strideq*0], m0
    pshuflw              m1, m0, q1032
    movd   [dstq+strideq*1], m1
    punpckhqdq           m0, m0
    movd   [dstq+strideq*2], m0
    psrlq                m0, 32
    movd   [dstq+r3       ], m0
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4_loop
    RET
ALIGN function_align
.w8:
    mova                 m7, [base+ipred_v_shuf]
    movq                 m1, [tlq+1]                  ; left
    punpcklqdq           m1, m1
    sub                 tlq, 4
    sub                 tlq, hq
    punpcklbw            m1, m0
    pshufd               m6, m7, q0000
    pshufd               m7, m7, q1111
    pmaddubsw            m2, m1, m5
    paddw                m3, m1
    paddw                m2, m3
    mova         [rsp+16*0], m1
    mova         [rsp+16*1], m2
    mova                 m1, [base+smooth_weights+8*2] ; weights_hor = &dav1d_sm_weights[width];
    mova         [rsp+16*2], m1
    mova         [rsp+16*3], m4
    mova         [rsp+16*4], m6
    mova         [rsp+16*5], m5
.w8_loop:
    movd                 m1, [tlq+hq]                  ; left
    pshufb               m1, [base+ipred_h_shuf]
    pshufd               m1, m1, q1100
    punpcklbw            m0, m1, m4
    punpckhbw            m1, m4
    pmaddubsw            m2, m0, m5
    pmaddubsw            m3, m1, m5
    paddw                m2, m0
    paddw                m3, m1
    mova                 m4,  [rsp+16*2]
    pmaddubsw            m0, m4
    pmaddubsw            m1, m4
    paddw                m2, m0
    paddw                m3, m1
    movd                 m1, [v_weightsq]              ; weights_ver = &dav1d_sm_weights[height];
    add          v_weightsq, 4
    pshufb               m0, m1, m6
    pshufb               m1, m7
    mova                 m4, [rsp+16*0]
    mova                 m5, [rsp+16*1]
    SMOOTH_2D_END 0, 1, 4, 4, 5, 5, 3
    mova                 m4, [rsp+16*3]
    mova                 m6, [rsp+16*4]
    mova                 m5, [rsp+16*5]
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w8_loop
    RET
ALIGN function_align
.w16:
    mova                 m7, [base+ipred_v_shuf]
    movu                 m1, [tlq+1]                     ; left
    sub                 tlq, 4
    sub                 tlq, hq
    punpckhbw            m6, m1, m0                      ; top, bottom
    punpcklbw            m1, m0                          ; top, bottom
    pshufd               m7, m7, q0000
    mova         [rsp+16*2], m7
    pmaddubsw            m2, m6, m5
    mova         [rsp+16*5], m6
    paddw                m6, m3                          ;   1 * top + 255 * bottom + 255
    paddw                m2, m6                          ; 128 * top + 129 * bottom + 255
    mova         [rsp+16*6], m2
    pmaddubsw            m2, m1, m5
    paddw                m3, m1                          ;   1 * top + 255 * bottom + 255
    mova         [rsp+16*0], m1
    paddw                m2, m3                          ; 128 * top + 129 * bottom + 255
    mova         [rsp+16*1], m2
    mova         [rsp+16*3], m4
    mova         [rsp+16*4], m5
.w16_loop:
    movd                 m1, [tlq+hq]                    ; left
    pshufb               m1, [base+pb_3]                 ; topleft[-(1 + y)]
    punpcklbw            m1, m4                          ; left, right
    pmaddubsw            m2, m1, m5                      ; 127 * left - 127 * right
    paddw                m2, m1                          ; 128 * left + 129 * right
    mova                 m0, m1
    mova                 m3, m2
    pmaddubsw            m0, [base+smooth_weights+16*2]  ; weights_hor = &dav1d_sm_weights[width];
    pmaddubsw            m1, [base+smooth_weights+16*3]
    paddw                m2, m0
    paddw                m3, m1
    movd                 m1, [v_weightsq]                ; weights_ver = &dav1d_sm_weights[height];
    add          v_weightsq, 2
    mova                 m7, [rsp+16*2]
    pshufb               m1, m7
    mova         [rsp+16*7], m3
    mova                 m4, [rsp+16*0]
    mova                 m5, [rsp+16*1]
    mova                 m3, [rsp+16*5]
    mova                 m7, [rsp+16*6]
    SMOOTH_2D_END 1, 1, 4, 3, 5, 7, [rsp+16*7]
    mova                 m4, [rsp+16*3]
    mova                 m5, [rsp+16*4]
    mova             [dstq], m0
    lea                dstq, [dstq+strideq]
    sub                  hd, 1
    jg .w16_loop
    RET
ALIGN function_align
.w32:
    movu                 m1, [tlq+1]                     ; top     topleft[1 + x]
    movu                 m2, [tlq+17]                    ; top
    mova         [rsp+16*0], m1
    mova         [rsp+16*1], m2
    sub                 tlq, 4
    sub                 tlq, hq
    mova                 m7, [base+ipred_v_shuf]
    pshufd               m7, m7, q0000
    mova         [rsp+16*2], m7
    mova         [rsp+16*3], m0
    mova         [rsp+16*4], m4
    mova         [rsp+16*5], m5
.w32_loop:
    SMOOTH_OUTPUT_16B 0, 6, 7, 8, 9, [base+smooth_weights+16*4], [base+smooth_weights+16*5], 10, 2, 3, 4, 5
    add                dstq, 16
    SMOOTH_OUTPUT_16B 1, 6, 7, 8, 9, [base+smooth_weights+16*6], [base+smooth_weights+16*7], 10, 2, 3, 4, 5
    lea                dstq, [dstq-16+strideq]
    add          v_weightsq, 2
    sub                  hd, 1
    jg .w32_loop
    RET
ALIGN function_align
.w64:
    movu                 m1, [tlq+1]                     ; top     topleft[1 + x]
    movu                 m2, [tlq+17]                    ; top
    mova         [rsp+16*0], m1
    mova         [rsp+16*1], m2
    movu                 m1, [tlq+33]                    ; top
    movu                 m2, [tlq+49]                    ; top
    mova        [rsp+16*11], m1
    mova        [rsp+16*12], m2
    sub                 tlq, 4
    sub                 tlq, hq
    mova                 m7, [base+ipred_v_shuf]
    pshufd               m7, m7, q0000
    mova         [rsp+16*2], m7
    mova         [rsp+16*3], m0
    mova         [rsp+16*4], m4
    mova         [rsp+16*5], m5
.w64_loop:
    SMOOTH_OUTPUT_16B  0, 6, 7, 8, 9,  [base+smooth_weights+16*8],  [base+smooth_weights+16*9], 10, 2, 3, 4, 5
    add                dstq, 16
    SMOOTH_OUTPUT_16B  1, 6, 7, 8, 9, [base+smooth_weights+16*10], [base+smooth_weights+16*11], 10, 2, 3, 4, 5
    add                dstq, 16
    SMOOTH_OUTPUT_16B 11, 6, 7, 8, 9, [base+smooth_weights+16*12], [base+smooth_weights+16*13], 10, 2, 3, 4, 5
    add                dstq, 16
    SMOOTH_OUTPUT_16B 12, 6, 7, 8, 9, [base+smooth_weights+16*14], [base+smooth_weights+16*15], 10, 2, 3, 4, 5
    lea                dstq, [dstq-48+strideq]
    add          v_weightsq, 2
    sub                  hd, 1
    jg .w64_loop
    RET

%if ARCH_X86_64
cglobal ipred_z1_8bpc, 3, 8, 11, 16*12, dst, stride, tl, w, h, angle, dx
    %define            base  r7-$$
    lea                  r7, [$$]
    movifnidn            hd, hm
    mova                 m8, [base+pw_62]
    mova                 m9, [base+pw_64]
    mova                m10, [base+pw_512]
%else
cglobal ipred_z1_8bpc, 3, 7, 8, -16*13, dst, stride, tl, w, h, angle, dx
    %define            base  r4-$$
    %define              m8  [base+pw_62]
    %define              m9  [base+pw_64]
    %define             m10  [base+pw_512]
    %define              hd  dword [rsp+16*12]
    %define              hb  byte [rsp+16*12]
    mov                  r3, hm
    LEA                  r4, $$
    mov                  hd, r3
%endif
    tzcnt                wd, wm
    movifnidn        angled, anglem
    inc                 tlq
    movsxd               wq, [base+ipred_z1_ssse3_table+wq*4]
    lea                  wq, [base+wq+ipred_z1_ssse3_table]
    mov                 dxd, angled
    and                 dxd, 0x7e
    add              angled, 165 ; ~90
    movzx               dxd, word [base+dr_intra_derivative+dxq]
    xor              angled, 0x4ff ; d = 90 - angle
    jmp                  wq
.w4:
%if ARCH_X86_64
    cmp              angleb, 40
%else
    mov                 r3d, angled ; rNb only valid for r0-r3 on x86-32
    cmp                 r3b, 40
%endif
    jae .w4_no_upsample
    lea                 r3d, [angleq-1024]
    sar                 r3d, 7
    add                 r3d, hd
    jg .w4_no_upsample ; !enable_intra_edge_filter || h > 8 || (h == 8 && is_sm)
    mova                 m1, [tlq-1]
    pshufb               m0, m1, [base+z_upsample1]
    pshufb               m1, [base+z_upsample2]
    movddup              m2, [base+pb_36_m4]
    add                 dxd, dxd
    pmaddubsw            m0, m2
    pshufd               m7, m1, q3333
    movd           [rsp+16], m7 ; top[max_base_x]
    pmaddubsw            m1, m2
    movd                 m6, dxd
    mov                 r3d, dxd ; xpos
    pshufb               m6, [base+pw_256]
    paddw                m1, m0
    movq                 m0, [tlq]
    pmulhrsw             m1, m10
    paddw                m7, m6, m6
    punpcklqdq           m6, m7 ; xpos0 xpos1
    packuswb             m1, m1
    punpcklbw            m0, m1
    mova              [rsp], m0
.w4_upsample_loop:
    lea                 r2d, [r3+dxq]
    shr                 r3d, 6      ; base0
    movq                 m0, [rsp+r3]
    lea                 r3d, [r2+dxq]
    shr                 r2d, 6      ; base1
    movhps               m0, [rsp+r2]
    pand                 m2, m8, m6 ; frac
    psubw                m1, m9, m2 ; 64-frac
    psllw                m2, 8
    por                  m1, m2     ; 64-frac, frac
    pmaddubsw            m0, m1
    paddw                m6, m7     ; xpos += dx
    pmulhrsw             m0, m10
    packuswb             m0, m0
    movd   [dstq+strideq*0], m0
    pshuflw              m0, m0, q1032
    movd   [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w4_upsample_loop
    RET
.w4_no_upsample:
    mov                 r3d, 7     ; max_base
    test             angled, 0x400 ; !enable_intra_edge_filter
    jnz .w4_main
%if ARCH_X86_64
    lea                 r3d, [hq+3]
%else
    mov                 r3d, hd
    add                 r3d, 3
%endif
    movd                 m0, r3d
    movd                 m2, angled
    shr              angled, 8 ; is_sm << 1
    pxor                 m1, m1
    pshufb               m0, m1
    pshufb               m2, m1
    pcmpeqb              m1, m0, [base+z_filter_wh4]
    pand                 m1, m2
    pcmpgtb              m1, [base+z_filter_t_w48+angleq*8]
    pmovmskb            r5d, m1
    mov                 r3d, 7
    test                r5d, r5d
    jz .w4_main ; filter_strength == 0
    mova                 m3, [tlq-1]
    imul                r5d, 0x55555555
    movu                 m7, [base+z_filter_s+8]
    shr                 r5d, 30 ; filter_strength
    movddup              m0, [base+pb_8]
    pminub               m7, m0
    pshufb               m0, m3, [base+z_filter_s]
    movddup              m4, [base+z_filter_k-8+r5*8+24*0]
    pshufb               m3, m7
    movddup              m5, [base+z_filter_k-8+r5*8+24*1]
    shufps               m2, m0, m3, q2121
    movddup              m6, [base+z_filter_k-8+r5*8+24*2]
    pmaddubsw            m0, m4
    pmaddubsw            m1, m2, m4
    pmaddubsw            m2, m5
    paddd                m5, m6
    pmaddubsw            m4, m3, m5
    pmaddubsw            m3, m6
    paddw                m0, m2
    paddw                m1, m4
    paddw                m0, m3
    pshufd               m1, m1, q3333
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    mov                 r5d, 9
    mov                 tlq, rsp
    cmp                  hd, 4
    cmovne              r3d, r5d
    packuswb             m0, m1
    mova              [tlq], m0
.w4_main:
    add                 tlq, r3
    movd                 m5, dxd
    movddup              m0, [base+z_base_inc] ; base_inc << 6
    movd                 m7, [tlq] ; top[max_base_x]
    shl                 r3d, 6
    movd                 m4, r3d
    pshufb               m5, [base+pw_256]
    mov                 r5d, dxd ; xpos
    pshufb               m7, [base+pw_m256]
    sub                  r5, r3
    pshufb               m4, [base+pw_256]
    mova                 m3, [base+z1_shuf_w4]
    paddw                m6, m5, m5
    psubw                m4, m0 ; max_base_x
    punpcklqdq           m5, m6 ; xpos0 xpos1
.w4_loop:
    lea                  r3, [r5+dxq]
    sar                  r5, 6      ; base0
    movq                 m0, [tlq+r5]
    lea                  r5, [r3+dxq]
    sar                  r3, 6      ; base1
    movhps               m0, [tlq+r3]
    pand                 m2, m8, m5 ; frac
    psubw                m1, m9, m2 ; 64-frac
    psllw                m2, 8
    pshufb               m0, m3
    por                  m1, m2     ; 64-frac, frac
    pmaddubsw            m0, m1
    pcmpgtw              m1, m4, m5 ; base < max_base_x
    pmulhrsw             m0, m10
    paddw                m5, m6     ; xpos += dx
    pand                 m0, m1
    pandn                m1, m7
    por                  m0, m1
    packuswb             m0, m0
    movd   [dstq+strideq*0], m0
    pshuflw              m0, m0, q1032
    movd   [dstq+strideq*1], m0
    sub                  hd, 2
    jz .w4_end
    lea                dstq, [dstq+strideq*2]
    test                r5d, r5d
    jl .w4_loop
    packuswb             m7, m7
.w4_end_loop:
    movd   [dstq+strideq*0], m7
    movd   [dstq+strideq*1], m7
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w4_end_loop
.w4_end:
    RET
.w8:
    lea                 r3d, [angleq+216]
    mov                 r3b, hb
    cmp                 r3d, 8
    ja .w8_no_upsample ; !enable_intra_edge_filter || is_sm || d >= 40 || h > 8
    mova                 m5, [base+z_upsample1]
    movu                 m3, [base+z_filter_s+6]
    movd                 m4, hd
    mova                 m0, [tlq-1]
    movu                 m1, [tlq+7]
    pxor                 m7, m7
    pshufb               m4, m7
    movddup              m7, [base+pb_36_m4]
    pminub               m4, m3
    add                 dxd, dxd
    pshufb               m2, m0, m5
    pmaddubsw            m2, m7
    pshufb               m0, m3
    pmaddubsw            m0, m7
    movd                 m6, dxd
    pshufb               m3, m1, m5
    pmaddubsw            m3, m7
    pshufb               m1, m4
    pmaddubsw            m1, m7
    pshufb               m6, [base+pw_256]
    mov                 r3d, dxd
    paddw                m2, m0
    paddw                m7, m6, m6
    paddw                m3, m1
    punpcklqdq           m6, m7 ; xpos0 xpos1
    movu                 m1, [tlq]
    pmulhrsw             m2, m10
    pmulhrsw             m3, m10
    packuswb             m2, m3
    punpcklbw            m0, m1, m2
    punpckhbw            m1, m2
    mova         [rsp+16*0], m0
    mova         [rsp+16*1], m1
.w8_upsample_loop:
    lea                 r2d, [r3+dxq]
    shr                 r3d, 6 ; base0
    movu                 m0, [rsp+r3]
    lea                 r3d, [r2+dxq]
    shr                 r2d, 6 ; base1
    movu                 m1, [rsp+r2]
    pand                 m2, m8, m6
    psubw                m3, m9, m2
    psllw                m2, 8
    por                  m3, m2
    punpcklqdq           m2, m3, m3 ; frac0
    pmaddubsw            m0, m2
    punpckhqdq           m3, m3     ; frac1
    pmaddubsw            m1, m3
    paddw                m6, m7
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    packuswb             m0, m1
    movq   [dstq+strideq*0], m0
    movhps [dstq+strideq*1], m0
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w8_upsample_loop
    RET
.w8_no_upsample:
%if ARCH_X86_64
    lea                 r3d, [hq+7]
%else
    mov                 r3d, hd
    add                 r3d, 7
%endif
    movd                 m0, r3d
    and                 r3d, 7
    or                  r3d, 8 ; imin(h+7, 15)
    test             angled, 0x400
    jnz .w8_main
    movd                 m2, angled
    shr              angled, 8 ; is_sm << 1
    pxor                 m1, m1
    pshufb               m0, m1
    pshufb               m2, m1
    movu                 m1, [base+z_filter_wh8]
    psrldq               m3, [base+z_filter_t_w48+angleq*8], 4
    pcmpeqb              m1, m0
    pand                 m1, m2
    pcmpgtb              m1, m3
    pmovmskb            r5d, m1
    test                r5d, r5d
    jz .w8_main ; filter_strength == 0
    mova                 m0, [tlq- 1]
    imul                r5d, 0x55555555
    mova                 m1, [tlq+15]
    shr                 r5d, 30 ; filter_strength
    movd                 m2, [tlq+r3]
    lea                 tlq, [rsp+16*4]
    sub                  r5, 3
    mova         [tlq-16*1], m0
    pxor                 m3, m3
    mova         [tlq+16*0], m1
    pshufb               m0, m3
    pshufb               m2, m3
    mova         [tlq-16*2], m0
    movq        [tlq+r3-15], m2
    call .filter_edge
    sar                 r5d, 1
    add                 r5d, 17
    cmp                  hd, 8
    cmova               r3d, r5d
.w8_main:
    add                 tlq, r3
    movd                 m5, dxd
    movd                 m7, [tlq]
    shl                 r3d, 6
    movu                 m3, [base+z_filter_s+2]
    movd                 m4, r3d
    pshufb               m5, [base+pw_256]
    mov                 r5d, dxd
    pshufb               m7, [base+pw_m256]
    sub                  r5, r3
    pshufb               m4, [base+pw_256]
    psubw                m4, [base+z_base_inc]
    mova                 m6, m5
.w8_loop:
    mov                  r3, r5
    sar                  r3, 6
    movu                 m0, [tlq+r3]
    pand                 m1, m8, m5
    psubw                m2, m9, m1
    psllw                m1, 8
    pshufb               m0, m3
    por                  m1, m2
    pmaddubsw            m0, m1
    pcmpgtw              m1, m4, m5
    paddw                m5, m6
    pmulhrsw             m0, m10
    pand                 m0, m1
    pandn                m1, m7
    por                  m0, m1
    packuswb             m0, m0
    movq             [dstq], m0
    dec                  hd
    jz .w8_end
    add                dstq, strideq
    add                  r5, dxq
    jl .w8_loop
    packuswb             m7, m7
.w8_end_loop:
    movq             [dstq], m7
    add                dstq, strideq
    dec                  hd
    jg .w8_end_loop
.w8_end:
    RET
.w16:
%if ARCH_X86_64
    lea                 r3d, [hq+15]
%else
    mov                 r3d, hd
    add                 r3d, 15
%endif
    movd                 m0, r3d
    and                 r3d, 15
    or                  r3d, 16 ; imin(h+15, 31)
    test             angled, 0x400
    jnz .w16_main
    movd                 m2, angled
    shr              angled, 8 ; is_sm << 1
    pxor                 m1, m1
    pshufb               m0, m1
    pshufb               m2, m1
    movq                 m3, [base+z_filter_t_w16+angleq*4]
    pcmpeqb              m1, m0, [base+z_filter_wh16]
    pand                 m1, m2
    pcmpgtb              m1, m3
    pmovmskb            r5d, m1
    test                r5d, r5d
    jz .w16_main ; filter_strength == 0
    mova                 m0, [tlq- 1]
    imul                r5d, 0x24924924
    mova                 m1, [tlq+15]
    shr                 r5d, 30
    movd                 m2, [tlq+30]
    adc                  r5, -4 ; filter_strength-3
    movd                 m3, [tlq+r3]
    lea                 tlq, [rsp+16*4]
    mova         [tlq-16*1], m0
    pxor                 m4, m4
    mova         [tlq+16*0], m1
    pshufb               m0, m4
    movd              [rsp], m2
    pshufb               m3, m4
    mova         [tlq-16*2], m0
    movd        [tlq+r3-15], m3
    call .filter_edge
    cmp                  hd, 16
    jle .w16_main
    pshuflw              m0, [rsp], q0000
    sar                  r5, 1
    movd                 m1, [base+z_filter_k_tail+4+r5*4]
    lea                 r3d, [r5+33]
    pmaddubsw            m0, m1
%if ARCH_X86_64
    pmulhrsw             m0, m10
%else
    pmulhrsw             m0, m4
%endif
    packuswb             m0, m0
    movd           [tlq+32], m0
.w16_main:
    add                 tlq, r3
    movd                 m5, dxd
    movd                 m7, [tlq]
    movd                 m4, r3d
    shl                 r3d, 6
    pshufb               m5, [base+pw_256]
    pxor                 m6, m6
    pshufb               m7, m6
    mov                 r5d, dxd
    pshufb               m4, m6
    sub                  r5, r3
    psubb                m4, [base+pb_0to15]
    mova                 m6, m5
.w16_loop:
    mov                  r3, r5
    sar                  r3, 6
    movu                 m1, [tlq+r3+0]
    pand                 m0, m8, m5
    movu                 m2, [tlq+r3+1]
    psubw                m3, m9, m0
    psllw                m0, 8
    por                  m3, m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    psrlw                m3, m5, 6
    packsswb             m3, m3
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    paddw                m5, m6
    pcmpgtb              m2, m4, m3
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    mova             [dstq], m0
    dec                  hd
    jz .w16_end
    add                dstq, strideq
    add                  r5, dxq
    jl .w16_loop
.w16_end_loop:
    mova             [dstq], m7
    add                dstq, strideq
    dec                  hd
    jg .w16_end_loop
.w16_end:
    RET
.w32:
%if ARCH_X86_64
    lea                 r3d, [hq+31]
%else
    mov                 r3d, hd
    add                 r3d, 31
%endif
    and                 r3d, 31
    or                  r3d, 32    ; imin(h+31, 63)
    test             angled, 0x400 ; !enable_intra_edge_filter
    jnz .w32_main

    mova                 m0, [tlq- 1]
    mova                 m1, [tlq+15]
    mova                 m2, [tlq+31]
    mova                 m3, [tlq+47]
    movd                 m4, [tlq+62]
    movd                 m5, [tlq+r3]
    lea                 tlq, [rsp+16*6]
    mova         [tlq-16*3], m0
    pxor                 m6, m6
    mova         [tlq-16*2], m1
    pshufb               m0, m6
    mova         [tlq-16*1], m2
    xor                 r5d, r5d ; filter_strength = 3
    mova         [tlq+16*0], m3
    movd              [rsp], m4
    pshufb               m5, m6
    mova         [tlq-16*4], m0
    movd        [tlq+r3-47], m5

    call .filter_edge
    sub                 tlq, 16*2
    call .filter_edge

    cmp                  hd, 32
    jle .w32_main
    pshuflw              m0, [rsp], q0000
    movd                 m1, [base+z_filter_k_tail+4]
    add                 r3d, 2
    pmaddubsw            m0, m1
%if ARCH_X86_64
    pmulhrsw             m0, m10
%else
    pmulhrsw             m0, m4
%endif
    packuswb             m0, m0
    movd           [tlq+64], m0
.w32_main:
    add                 tlq, r3
    movd                 m0, r3d
    movd                 m7, [tlq]
    shl                 r3d, 6
    movd                 m5, dxd
    pxor                 m6, m6
    mov                 r5d, dxd
    pshufb               m0, m6
    pshufb               m5, [base+pw_256]
    sub                  r5, r3
    pshufb               m7, m6
    psubb                m0, [base+pb_0to15]
    movddup              m1, [base+pb_m16]
    mova         [rsp+16*0], m0
    paddb                m0, m1
    mova         [rsp+16*1], m0
    mova                 m6, m5
.w32_loop:
    mov                  r3, r5
    sar                  r3, 6
    movu                 m1, [tlq+r3+16*0+0]
    pand                 m0, m8, m5
    movu                 m2, [tlq+r3+16*0+1]
    psubw                m3, m9, m0
    psllw                m0, 8
    por                  m3, m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    psrlw                m4, m5, 6
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    packsswb             m4, m4
    pcmpgtb              m2, [rsp+16*0], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    movu                 m1, [tlq+r3+16*1+0]
    movu                 m2, [tlq+r3+16*1+1]
    mova        [dstq+16*0], m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    paddw                m5, m6
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    pcmpgtb              m2, [rsp+16*1], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    mova        [dstq+16*1], m0
    dec                  hd
    jz .w32_end
    add                dstq, strideq
    add                  r5, dxq
    jl .w32_loop
.w32_end_loop:
    mova        [dstq+16*0], m7
    mova        [dstq+16*1], m7
    add                dstq, strideq
    dec                  hd
    jg .w32_end_loop
.w32_end:
    RET
.w64:
%if ARCH_X86_64
    lea                 r3d, [hq+63]
%else
    mov                 r3d, hd
    add                 r3d, 63
%endif
    test             angled, 0x400 ; !enable_intra_edge_filter
    jnz .w64_main
    mova                 m0, [tlq-  1]
    mova                 m1, [tlq+ 15]
    mova                 m2, [tlq+ 31]
    mova                 m3, [tlq+ 47]
    mova         [rsp+16*3], m0
    pxor                 m5, m5
    mova         [rsp+16*4], m1
    pshufb               m0, m5
    mova         [rsp+16*5], m2
    mova         [rsp+16*6], m3
    mova         [rsp+16*2], m0
    mova                 m0, [tlq+ 63]
    mova                 m1, [tlq+ 79]
    mova                 m2, [tlq+ 95]
    mova                 m3, [tlq+111]
    movd                 m4, [tlq+r3]
    lea                 tlq, [rsp+16*10]
    mova         [tlq-16*3], m0
    xor                 r5d, r5d ; filter_strength = 3
    mova         [tlq-16*2], m1
    pshufb               m4, m5
    mova         [tlq-16*1], m2
    mova         [tlq+16*0], m3
    movd       [tlq+r3-111], m4
    cmp                  hd, 64
    jl .w64_filter96 ; skip one call if the last 32 bytes aren't used
    call .filter_edge
.w64_filter96:
    sub                 tlq, 16*2
    call .filter_edge
    sub                 tlq, 16*2
    call .filter_edge
    sub                 tlq, 16*2
    call .filter_edge
.w64_main:
    add                 tlq, r3
    movd                 m0, r3d
    movd                 m7, [tlq]
    shl                 r3d, 6
    movd                 m5, dxd
    pxor                 m6, m6
    mov                 r5d, dxd
    pshufb               m0, m6
    sub                  r5, r3
    pshufb               m5, [base+pw_256]
    pshufb               m7, m6
    psubb                m0, [base+pb_0to15]
    movddup              m1, [base+pb_m16]
    mova         [rsp+16*0], m0
    paddb                m0, m1
    mova         [rsp+16*1], m0
    paddb                m0, m1
    mova         [rsp+16*2], m0
    paddb                m0, m1
    mova         [rsp+16*3], m0
    mova                 m6, m5
.w64_loop:
    mov                  r3, r5
    sar                  r3, 6
    movu                 m1, [tlq+r3+16*0+0]
    pand                 m0, m8, m5
    movu                 m2, [tlq+r3+16*0+1]
    psubw                m3, m9, m0
    psllw                m0, 8
    por                  m3, m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    psrlw                m4, m5, 6
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    packsswb             m4, m4
    pcmpgtb              m2, [rsp+16*0], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    movu                 m1, [tlq+r3+16*1+0]
    movu                 m2, [tlq+r3+16*1+1]
    mova        [dstq+16*0], m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    pcmpgtb              m2, [rsp+16*1], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    movu                 m1, [tlq+r3+16*2+0]
    movu                 m2, [tlq+r3+16*2+1]
    mova        [dstq+16*1], m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    pcmpgtb              m2, [rsp+16*2], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    movu                 m1, [tlq+r3+16*3+0]
    movu                 m2, [tlq+r3+16*3+1]
    mova        [dstq+16*2], m0
    punpcklbw            m0, m1, m2
    pmaddubsw            m0, m3
    punpckhbw            m1, m2
    pmaddubsw            m1, m3
    paddw                m5, m6
    pmulhrsw             m0, m10
    pmulhrsw             m1, m10
    pcmpgtb              m2, [rsp+16*3], m4
    packuswb             m0, m1
    pand                 m0, m2
    pandn                m2, m7
    por                  m0, m2
    mova        [dstq+16*3], m0
    dec                  hd
    jz .w64_end
    add                dstq, strideq
    add                  r5, dxq
    jl .w64_loop
.w64_end_loop:
    mova        [dstq+16*0], m7
    mova        [dstq+16*1], m7
    mova        [dstq+16*2], m7
    mova        [dstq+16*3], m7
    add                dstq, strideq
    dec                  hd
    jg .w64_end_loop
.w64_end:
    RET
ALIGN function_align
.filter_edge: ; 32 pixels/iteration
    movddup              m7, [base+z_filter_k+8*2+r5*8+24*0]
    movu                 m2, [tlq-17]
    mova                 m1, [tlq-16]
    movu                 m3, [tlq- 1]
    mova                 m4, [tlq+ 0]
    punpcklbw            m0, m2, m1
    pmaddubsw            m0, m7
    punpckhbw            m2, m1
    pmaddubsw            m2, m7
    punpcklbw            m1, m3, m4
    pmaddubsw            m1, m7
    punpckhbw            m3, m4
    pmaddubsw            m3, m7
    movddup              m7, [base+z_filter_k+8*2+r5*8+24*1]
    movu                 m5, [tlq-15]
    movu                 m6, [tlq-14]
    punpcklbw            m4, m5, m6
    pmaddubsw            m4, m7
    punpckhbw            m5, m6
    pmaddubsw            m5, m7
    paddw                m0, m4
    paddw                m2, m5
    movu                 m5, [tlq+ 1]
    movu                 m6, [tlq+ 2]
    punpcklbw            m4, m5, m6
    pmaddubsw            m4, m7
    punpckhbw            m5, m6
    pmaddubsw            m5, m7
    paddw                m1, m4
    paddw                m3, m5
    test                r5d, r5d
    jnz .filter_end ; 3-tap
    movddup              m7, [base+z_filter_k+8*8]
    movu                 m5, [tlq-13]
    movu                 m6, [tlq+ 3]
    punpcklbw            m4, m5, m5
    pmaddubsw            m4, m7
    punpckhbw            m5, m5
    pmaddubsw            m5, m7
    paddw                m0, m4
    paddw                m2, m5
    punpcklbw            m5, m6, m6
    pmaddubsw            m5, m7
    punpckhbw            m6, m6
    pmaddubsw            m6, m7
    paddw                m1, m5
    paddw                m3, m6
.filter_end:
%if ARCH_X86_64
    REPX  {pmulhrsw x, m10}, m0, m2, m1, m3
%else
    mova                 m4, m10
    REPX  {pmulhrsw x, m4 }, m0, m2, m1, m3
%endif
    packuswb             m0, m2
    packuswb             m1, m3
    mova         [tlq+16*0], m0
    mova         [tlq+16*1], m1
    ret

;---------------------------------------------------------------------------------------
;int dav1d_pal_pred_ssse3(pixel *dst, const ptrdiff_t stride, const uint16_t *const pal,
;                                         const uint8_t *idx, const int w, const int h);
;---------------------------------------------------------------------------------------
cglobal pal_pred_8bpc, 4, 6, 5, dst, stride, pal, idx, w, h
    mova                 m4, [palq]
    LEA                  r2, pal_pred_ssse3_table
    tzcnt                wd, wm
    movifnidn            hd, hm
    movsxd               wq, [r2+wq*4]
    packuswb             m4, m4
    add                  wq, r2
    lea                  r2, [strideq*3]
    jmp                  wq
.w4:
    pshufb               m0, m4, [idxq]
    add                idxq, 16
    movd   [dstq          ], m0
    pshuflw              m1, m0, q1032
    movd   [dstq+strideq  ], m1
    punpckhqdq           m0, m0
    movd   [dstq+strideq*2], m0
    psrlq                m0, 32
    movd   [dstq+r2       ], m0
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4
    RET
ALIGN function_align
.w8:
    pshufb               m0, m4, [idxq]
    pshufb               m1, m4, [idxq+16]
    add                idxq, 32
    movq   [dstq          ], m0
    movhps [dstq+strideq  ], m0
    movq   [dstq+strideq*2], m1
    movhps [dstq+r2       ], m1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w8
    RET
ALIGN function_align
.w16:
    pshufb               m0, m4, [idxq]
    pshufb               m1, m4, [idxq+16]
    pshufb               m2, m4, [idxq+32]
    pshufb               m3, m4, [idxq+48]
    add                idxq, 64
    mova   [dstq          ], m0
    mova   [dstq+strideq  ], m1
    mova   [dstq+strideq*2], m2
    mova   [dstq+r2       ], m3
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w16
    RET
ALIGN function_align
.w32:
    pshufb               m0, m4, [idxq]
    pshufb               m1, m4, [idxq+16]
    pshufb               m2, m4, [idxq+32]
    pshufb               m3, m4, [idxq+48]
    add                idxq, 64
    mova  [dstq           ], m0
    mova  [dstq+16        ], m1
    mova  [dstq+strideq   ], m2
    mova  [dstq+strideq+16], m3
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w32
    RET
ALIGN function_align
.w64:
    pshufb               m0, m4, [idxq]
    pshufb               m1, m4, [idxq+16]
    pshufb               m2, m4, [idxq+32]
    pshufb               m3, m4, [idxq+48]
    add                idxq, 64
    mova          [dstq   ], m0
    mova          [dstq+16], m1
    mova          [dstq+32], m2
    mova          [dstq+48], m3
    add                dstq, strideq
    sub                  hd, 1
    jg .w64
    RET

;---------------------------------------------------------------------------------------
;void dav1d_ipred_cfl_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                           const int width, const int height, const int16_t *ac, const int alpha);
;---------------------------------------------------------------------------------------
%macro IPRED_CFL 1                   ; ac in, unpacked pixels out
    psignw               m3, m%1, m1
    pabsw               m%1, m%1
    pmulhrsw            m%1, m2
    psignw              m%1, m3
    paddw               m%1, m0
%endmacro

%if UNIX64
DECLARE_REG_TMP 7
%else
DECLARE_REG_TMP 5
%endif

cglobal ipred_cfl_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha
    movifnidn            wd, wm
    movifnidn            hd, hm
    tzcnt               r6d, hd
    lea                 t0d, [wq+hq]
    movd                 m4, t0d
    tzcnt               t0d, t0d
    movd                 m5, t0d
    LEA                  t0, ipred_cfl_ssse3_table
    tzcnt                wd, wd
    movsxd               r6, [t0+r6*4]
    movsxd               wq, [t0+wq*4+16]
    pcmpeqd              m3, m3
    psrlw                m4, 1
    add                  r6, t0
    add                  wq, t0
    movifnidn           acq, acmp
    jmp                  r6
.h4:
    movd                 m0, [tlq-4]
    pmaddubsw            m0, m3
    jmp                  wq
.w4:
    movd                 m1, [tlq+1]
    pmaddubsw            m1, m3
    psubw                m0, m4
    paddw                m0, m1
    pmaddwd              m0, m3
    cmp                  hd, 4
    jg .w4_mul
    psrlw                m0, 3                             ; dc >>= ctz(width + height);
    jmp .w4_end
.w4_mul:
    punpckhqdq           m1, m0, m0
    paddw                m0, m1
    pshuflw              m1, m0, q1032                     ; psrlq                m1, m0, 32
    paddw                m0, m1
    psrlw                m0, 2
    mov                 r6d, 0x5556
    mov                 r2d, 0x3334
    test                 hd, 8
    cmovz               r6d, r2d
    movd                 m5, r6d
    pmulhuw              m0, m5
.w4_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s4:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    lea                  r6, [strideq*3]
    pabsw                m2, m1
    psllw                m2, 9
.s4_loop:
    mova                 m4, [acq]
    mova                 m5, [acq+16]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    movd   [dstq+strideq*0], m4
    pshuflw              m4, m4, q1032
    movd   [dstq+strideq*1], m4
    punpckhqdq           m4, m4
    movd   [dstq+strideq*2], m4
    psrlq                m4, 32
    movd   [dstq+r6       ], m4
    lea                dstq, [dstq+strideq*4]
    add                 acq, 32
    sub                  hd, 4
    jg .s4_loop
    RET
ALIGN function_align
.h8:
    movq                 m0, [tlq-8]
    pmaddubsw            m0, m3
    jmp                  wq
.w8:
    movq                 m1, [tlq+1]
    pmaddubsw            m1, m3
    psubw                m4, m0
    punpckhqdq           m0, m0
    psubw                m0, m4
    paddw                m0, m1
    pshuflw              m1, m0, q1032                  ; psrlq  m1, m0, 32
    paddw                m0, m1
    pmaddwd              m0, m3
    psrlw                m0, m5
    cmp                  hd, 8
    je .w8_end
    mov                 r6d, 0x5556
    mov                 r2d, 0x3334
    cmp                  hd, 32
    cmovz               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
.w8_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s8:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    lea                  r6, [strideq*3]
    pabsw                m2, m1
    psllw                m2, 9
.s8_loop:
    mova                 m4, [acq]
    mova                 m5, [acq+16]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    movq   [dstq          ], m4
    movhps [dstq+strideq  ], m4
    mova                 m4, [acq+32]
    mova                 m5, [acq+48]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    movq   [dstq+strideq*2], m4
    movhps [dstq+r6       ], m4
    lea                dstq, [dstq+strideq*4]
    add                 acq, 64
    sub                  hd, 4
    jg .s8_loop
    RET
ALIGN function_align
.h16:
    mova                 m0, [tlq-16]
    pmaddubsw            m0, m3
    jmp                  wq
.w16:
    movu                 m1, [tlq+1]
    pmaddubsw            m1, m3
    paddw                m0, m1
    psubw                m4, m0
    punpckhqdq           m0, m0
    psubw                m0, m4
    pshuflw              m1, m0, q1032                  ; psrlq  m1, m0, 32
    paddw                m0, m1
    pmaddwd              m0, m3
    psrlw                m0, m5
    cmp                  hd, 16
    je .w16_end
    mov                 r6d, 0x5556
    mov                 r2d, 0x3334
    test                 hd, 8|32
    cmovz               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
.w16_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s16:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s16_loop:
    mova                 m4, [acq]
    mova                 m5, [acq+16]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    mova             [dstq], m4
    mova                 m4, [acq+32]
    mova                 m5, [acq+48]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    mova     [dstq+strideq], m4
    lea                dstq, [dstq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .s16_loop
    RET
ALIGN function_align
.h32:
    mova                 m0, [tlq-32]
    pmaddubsw            m0, m3
    mova                 m2, [tlq-16]
    pmaddubsw            m2, m3
    paddw                m0, m2
    jmp                  wq
.w32:
    movu                 m1, [tlq+1]
    pmaddubsw            m1, m3
    movu                 m2, [tlq+17]
    pmaddubsw            m2, m3
    paddw                m1, m2
    paddw                m0, m1
    psubw                m4, m0
    punpckhqdq           m0, m0
    psubw                m0, m4
    pshuflw              m1, m0, q1032                   ; psrlq  m1, m0, 32
    paddw                m0, m1
    pmaddwd              m0, m3
    psrlw                m0, m5
    cmp                  hd, 32
    je .w32_end
    lea                 r2d, [hq*2]
    mov                 r6d, 0x5556
    mov                 r2d, 0x3334
    test                 hd, 64|16
    cmovz               r6d, r2d
    movd                 m1, r6d
    pmulhuw              m0, m1
.w32_end:
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
.s32:
    movd                 m1, alpham
    pshuflw              m1, m1, q0000
    punpcklqdq           m1, m1
    pabsw                m2, m1
    psllw                m2, 9
.s32_loop:
    mova                 m4, [acq]
    mova                 m5, [acq+16]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    mova             [dstq], m4
    mova                 m4, [acq+32]
    mova                 m5, [acq+48]
    IPRED_CFL             4
    IPRED_CFL             5
    packuswb             m4, m5
    mova          [dstq+16], m4
    add                dstq, strideq
    add                 acq, 64
    dec                  hd
    jg .s32_loop
    RET

;---------------------------------------------------------------------------------------
;void dav1d_ipred_cfl_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                           const int width, const int height, const int16_t *ac, const int alpha);
;---------------------------------------------------------------------------------------
cglobal ipred_cfl_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha
    mov                  hd, hm                                 ; zero upper half
    tzcnt               r6d, hd
    sub                 tlq, hq
    tzcnt                wd, wm
    movu                 m0, [tlq]
    mov                 t0d, 0x8000
    movd                 m3, t0d
    movd                 m2, r6d
    psrld                m3, m2
    LEA                  t0, ipred_cfl_left_ssse3_table
    movsxd               r6, [t0+r6*4]
    pcmpeqd              m2, m2
    pmaddubsw            m0, m2
    add                  r6, t0
    add                  t0, ipred_cfl_splat_ssse3_table-ipred_cfl_left_ssse3_table
    movsxd               wq, [t0+wq*4]
    add                  wq, t0
    movifnidn           acq, acmp
    jmp                  r6
.h32:
    movu                 m1, [tlq+16]                           ; unaligned when jumping here from dc_top
    pmaddubsw            m1, m2
    paddw                m0, m1
.h16:
    pshufd               m1, m0, q3232                          ; psrlq               m1, m0, 16
    paddw                m0, m1
.h8:
    pshuflw              m1, m0, q1032                          ; psrlq               m1, m0, 32
    paddw                m0, m1
.h4:
    pmaddwd              m0, m2
    pmulhrsw             m0, m3
    pshuflw              m0, m0, q0000
    punpcklqdq           m0, m0
    jmp                  wq

;---------------------------------------------------------------------------------------
;void dav1d_ipred_cfl_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                           const int width, const int height, const int16_t *ac, const int alpha);
;---------------------------------------------------------------------------------------
cglobal ipred_cfl_top_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha
    LEA                  t0, ipred_cfl_left_ssse3_table
    tzcnt                wd, wm
    inc                 tlq
    movu                 m0, [tlq]
    movifnidn            hd, hm
    mov                 r6d, 0x8000
    movd                 m3, r6d
    movd                 m2, wd
    psrld                m3, m2
    movsxd               r6, [t0+wq*4]
    pcmpeqd              m2, m2
    pmaddubsw            m0, m2
    add                  r6, t0
    add                  t0, ipred_cfl_splat_ssse3_table-ipred_cfl_left_ssse3_table
    movsxd               wq, [t0+wq*4]
    add                  wq, t0
    movifnidn           acq, acmp
    jmp                  r6

;---------------------------------------------------------------------------------------
;void dav1d_ipred_cfl_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft,
;                           const int width, const int height, const int16_t *ac, const int alpha);
;---------------------------------------------------------------------------------------
cglobal ipred_cfl_128_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha
    tzcnt                wd, wm
    movifnidn            hd, hm
    LEA                  r6, ipred_cfl_splat_ssse3_table
    movsxd               wq, [r6+wq*4]
    movddup              m0, [r6-ipred_cfl_splat_ssse3_table+pw_128]
    add                  wq, r6
    movifnidn           acq, acmp
    jmp                  wq

%macro RELOAD_ACQ_32 1
    mov                 acq, ac_bakq       ; restore acq
%endmacro

%if ARCH_X86_64
cglobal ipred_cfl_ac_420_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak
DECLARE_REG_TMP 7
    movddup              m2, [pb_2]
%else
cglobal ipred_cfl_ac_420_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h
DECLARE_REG_TMP 4
%define ac_bakq acmp
    mov                 t0d, 0x02020202
    movd                 m2, t0d
    pshufd               m2, m2, q0000
%endif
    movifnidn            wd, wm
    mov                 t0d, hm
    mov                  hd, t0d
    imul                t0d, wd
    movd                 m5, t0d
    movifnidn         hpadd, hpadm
%if ARCH_X86_64
    mov             ac_bakq, acq
%endif
    shl               hpadd, 2
    sub                  hd, hpadd
    pxor                 m4, m4
    cmp                  wd, 8
    jg .w16
    je .w8
    ; fall-through
%if ARCH_X86_64
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak
%else
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h
%endif
.w4:
    lea            stride3q, [strideq*3]
.w4_loop:
    movq                 m0, [yq]
    movq                 m1, [yq+strideq]
    movhps               m0, [yq+strideq*2]
    movhps               m1, [yq+stride3q]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova              [acq], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*4]
    add                 acq, 16
    sub                  hd, 2
    jg .w4_loop
    test              hpadd, hpadd
    jz .calc_avg_4_8
    punpckhqdq           m0, m0
.w4_hpad_loop:
    mova              [acq], m0
    paddw                m4, m0
    add                 acq, 16
    sub               hpadd, 2
    jg .w4_hpad_loop
    jmp .calc_avg_4_8
.w8:
    lea            stride3q, [strideq*3]
    test              wpadd, wpadd
    jnz .w8_wpad
.w8_loop:
    mova                 m0, [yq]
    mova                 m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova              [acq], m0
    paddw                m4, m0
    mova                 m0, [yq+strideq*2]
    mova                 m1, [yq+stride3q]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova           [acq+16], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*4]
    add                 acq, 32
    sub                  hd, 2
    jg .w8_loop
    test              hpadd, hpadd
    jz .calc_avg_4_8
    jmp .w8_hpad
.w8_wpad:                                              ; wpadd=1
    movddup              m0, [yq]
    movddup              m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    pshufhw              m0, m0, q3333
    mova              [acq], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 16
    sub                  hd, 1
    jg .w8_wpad
    test              hpadd, hpadd
    jz .calc_avg_4_8
.w8_hpad:
    mova              [acq], m0
    paddw                m4, m0
    add                 acq, 16
    sub               hpadd, 1
    jg .w8_hpad
    jmp .calc_avg_4_8
.w16:
    test              wpadd, wpadd
    jnz .w16_wpad
.w16_loop:
    mova                 m0, [yq]
    mova                 m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova              [acq], m0
    paddw                m4, m0
    mova                 m6, [yq+16]
    mova                 m1, [yq+strideq+16]
    pmaddubsw            m6, m2
    pmaddubsw            m1, m2
    paddw                m6, m1
    mova           [acq+16], m6
    paddw                m4, m6
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    dec                  hd
    jg .w16_loop
    test              hpadd, hpadd
    jz .calc_avg16
    jmp .w16_hpad_loop
.w16_wpad:
    cmp               wpadd, 2
    jl .w16_pad1
    je .w16_pad2
.w16_pad3:
    movddup              m0, [yq]
    movddup              m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    pshufhw              m0, m0, q3333
    mova              [acq], m0
    paddw                m4, m0
    mova                 m6, m0
    punpckhqdq           m6, m0, m0
    mova           [acq+16], m6
    paddw                m4, m6
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    dec                  hd
    jg .w16_pad3
    jmp .w16_wpad_done
.w16_pad2:
    mova                 m0, [yq]
    mova                 m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova              [acq], m0
    paddw                m4, m0
    pshufhw              m6, m0, q3333
    punpckhqdq           m6, m6
    mova           [acq+16], m6
    paddw                m4, m6
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    dec                  hd
    jg .w16_pad2
    jmp .w16_wpad_done
.w16_pad1:
    mova                 m0, [yq]
    mova                 m1, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    paddw                m0, m1
    mova              [acq], m0
    paddw                m4, m0
    movddup              m6, [yq+16]
    movddup              m1, [yq+strideq+16]
    pmaddubsw            m6, m2
    pmaddubsw            m1, m2
    paddw                m6, m1
    pshufhw              m6, m6, q3333
    mova           [acq+16], m6
    paddw                m4, m6
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    dec                  hd
    jg .w16_pad1
.w16_wpad_done:
    test              hpadd, hpadd
    jz .calc_avg16
.w16_hpad_loop:
    mova              [acq], m0
    paddw                m4, m0
    mova           [acq+16], m6
    paddw                m4, m6
    add                 acq, 32
    dec               hpadd
    jg .w16_hpad_loop
    jmp .calc_avg16

%if ARCH_X86_64
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak
%else
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h
%endif
.calc_avg_4_8:
    psrlw                m2, 9
    pmaddwd              m4, m2
    jmp .calc_avg
.calc_avg16:
    psrld                m0, m4, 16
    pslld                m4, 16
    psrld                m4, 16
    paddd                m4, m0
.calc_avg:
    movd                szd, m5
    psrad                m5, 1
    tzcnt               r1d, szd
    paddd                m4, m5
    movd                 m1, r1d
    pshufd               m0, m4, q2301
    paddd                m0, m4
    pshufd               m4, m0, q1032
    paddd                m0, m4
    psrad                m0, m1                        ; sum >>= log2sz;
    packssdw             m0, m0
    RELOAD_ACQ_32       acq
.sub_loop:
    mova                 m1, [acq]
    psubw                m1, m0                        ; ac[x] -= sum;
    mova              [acq], m1
    add                 acq, 16
    sub                 szd, 8
    jg .sub_loop
    RET

%if ARCH_X86_64
cglobal ipred_cfl_ac_422_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak
    movddup              m2, [pb_4]
%else
cglobal ipred_cfl_ac_422_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h
    mov                 t0d, 0x04040404
    movd                 m2, t0d
    pshufd               m2, m2, q0000
%endif
    movifnidn            wd, wm
    mov                 t0d, hm
    mov                  hd, t0d
    imul                t0d, wd
    movd                 m6, t0d
    movifnidn         hpadd, hpadm
%if ARCH_X86_64
    mov             ac_bakq, acq
%endif
    shl               hpadd, 2
    sub                  hd, hpadd
    pxor                 m4, m4
    pxor                 m5, m5
    cmp                  wd, 8
    jg .w16
    je .w8
    ; fall-through

%if ARCH_X86_64
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak
%else
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h
%endif
.w4:
    lea            stride3q, [strideq*3]
.w4_loop:
    movq                 m1, [yq]
    movhps               m1, [yq+strideq]
    movq                 m0, [yq+strideq*2]
    movhps               m0, [yq+stride3q]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m4, m0
    paddw                m5, m1
    lea                  yq, [yq+strideq*4]
    add                 acq, 32
    sub                  hd, 4
    jg .w4_loop
    test              hpadd, hpadd
    jz .calc_avg_4
    punpckhqdq           m0, m0
.w4_hpad_loop:
    mova              [acq], m0
    paddw                m4, m0
    add                 acq, 16
    sub               hpadd, 2
    jg .w4_hpad_loop
    jmp .calc_avg_4
.w8:
    lea            stride3q, [strideq*3]
    test              wpadd, wpadd
    jnz .w8_wpad
.w8_loop:
    mova                 m1, [yq]
    mova                 m0, [yq+strideq]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m4, m0
    paddw                m5, m1
    mova                 m1, [yq+strideq*2]
    mova                 m0, [yq+stride3q]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    mova           [acq+48], m0
    paddw                m4, m0
    paddw                m5, m1
    lea                  yq, [yq+strideq*4]
    add                 acq, 64
    sub                  hd, 4
    jg .w8_loop
    test              hpadd, hpadd
    jz .calc_avg_8_16
    jmp .w8_hpad
.w8_wpad:
    movddup              m1, [yq]
    pmaddubsw            m1, m2
    pshufhw              m1, m1, q3333
    mova              [acq], m1
    paddw                m5, m1
    movddup              m0, [yq+strideq]
    pmaddubsw            m0, m2
    pshufhw              m0, m0, q3333
    mova           [acq+16], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    sub                  hd, 2
    jg .w8_wpad
    test              hpadd, hpadd
    jz .calc_avg_8_16
.w8_hpad:
    mova              [acq], m0
    paddw                m4, m0
    mova           [acq+16], m0
    paddw                m4, m0
    add                 acq, 32
    sub               hpadd, 2
    jg .w8_hpad
    jmp .calc_avg_8_16
.w16:
    test              wpadd, wpadd
    jnz .w16_wpad
.w16_loop:
    mova                 m1, [yq]
    mova                 m0, [yq+16]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m5, m0
    paddw                m5, m1
    mova                 m1, [yq+strideq]
    mova                 m0, [yq+strideq+16]
    pmaddubsw            m0, m2
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    mova           [acq+48], m0
    paddw                m4, m0
    paddw                m4, m1
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_loop
    test              hpadd, hpadd
    jz .calc_avg_8_16
    jmp .w16_hpad_loop
.w16_wpad:
    cmp               wpadd, 2
    jl .w16_pad1
    je .w16_pad2
.w16_pad3:
    movddup              m1, [yq]
    pmaddubsw            m1, m2
    pshufhw              m1, m1, q3333
    mova              [acq], m1
    paddw                m5, m1
    punpckhqdq           m1, m1
    mova           [acq+16], m1
    paddw                m5, m1
    movddup              m1, [yq+strideq]
    pmaddubsw            m1, m2
    pshufhw              m1, m1, q3333
    mova           [acq+32], m1
    paddw                m4, m1
    punpckhqdq           m0, m1, m1
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad3
    jmp .w16_wpad_done
.w16_pad2:
    mova                 m1, [yq]
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    pshufhw              m1, m1, q3333
    punpckhqdq           m1, m1
    mova           [acq+16], m1
    paddw                m5, m1
    mova                 m1, [yq+strideq]
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    mova                 m0, m1
    pshufhw              m0, m0, q3333
    punpckhqdq           m0, m0
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad2
    jmp .w16_wpad_done
.w16_pad1:
    mova                 m1, [yq]
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    movddup              m0, [yq+16]
    pmaddubsw            m0, m2
    pshufhw              m0, m0, q3333
    mova           [acq+16], m0
    paddw                m5, m0
    mova                 m1, [yq+strideq]
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    movddup              m0, [yq+strideq+16]
    pmaddubsw            m0, m2
    pshufhw              m0, m0, q3333
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad1
.w16_wpad_done:
    test              hpadd, hpadd
    jz .calc_avg_8_16
.w16_hpad_loop:
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m4, m1
    paddw                m5, m0
    mova           [acq+32], m1
    mova           [acq+48], m0
    paddw                m4, m1
    paddw                m5, m0
    add                 acq, 64
    sub               hpadd, 2
    jg .w16_hpad_loop
    jmp .calc_avg_8_16

%if ARCH_X86_64
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak
%else
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h
%endif
.calc_avg_4:
    psrlw                m2, 10
    pmaddwd              m5, m2
    pmaddwd              m0, m4, m2
    jmp .calc_avg
.calc_avg_8_16:
    mova                 m0, m5
    psrld                m5, 16
    pslld                m0, 16
    psrld                m0, 16
    paddd                m5, m0
    mova                 m0, m4
    psrld                m0, 16
    pslld                m4, 16
    psrld                m4, 16
    paddd                m0, m4
.calc_avg:
    paddd                m5, m0
    movd                szd, m6
    psrad                m6, 1
    tzcnt               r1d, szd                       ; const int log2sz = ctz(width) + ctz(height);
    paddd                m5, m6
    movd                 m1, r1d
    pshufd               m0, m5, q2301
    paddd                m0, m5
    pshufd               m5, m0, q1032
    paddd                m0, m5
    psrad                m0, m1                        ; sum >>= log2sz;
    packssdw             m0, m0
    RELOAD_ACQ_32       acq                            ; ac = ac_orig
.sub_loop:
    mova                 m1, [acq]
    psubw                m1, m0
    mova              [acq], m1
    add                 acq, 16
    sub                 szd, 8
    jg .sub_loop
    RET

%if ARCH_X86_64
cglobal ipred_cfl_ac_444_8bpc, 4, 8, 7, -4*16, ac, y, stride, wpad, hpad, w, h, ac_bak
    movddup              m2, [pb_4]
%else
cglobal ipred_cfl_ac_444_8bpc, 4, 7, 7, -5*16, ac, y, stride, wpad, hpad, w, h
%define ac_bakq [rsp+16*4]
    mov                 t0d, 0x04040404
    movd                 m2, t0d
    pshufd               m2, m2, q0000
%endif
    movifnidn            wd, wm
    movifnidn         hpadd, hpadm
    movd                 m0, hpadd
    mov                 t0d, hm
    mov                  hd, t0d
    imul                t0d, wd
    movd                 m6, t0d
    movd              hpadd, m0
    mov             ac_bakq, acq
    shl               hpadd, 2
    sub                  hd, hpadd
    pxor                 m5, m5
    pxor                 m4, m4
    cmp                  wd, 16
    jg .w32
    cmp                  wd, 8
    jg .w16
    je .w8
    ; fall-through

%if ARCH_X86_64
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, ac_bak
%else
    DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h
%endif
.w4:
    lea            stride3q, [strideq*3]
.w4_loop:
    movd                 m1, [yq]
    movd                 m3, [yq+strideq]
    punpckldq            m1, m3
    punpcklbw            m1, m1
    movd                 m0, [yq+strideq*2]
    movd                 m3, [yq+stride3q]
    punpckldq            m0, m3
    punpcklbw            m0, m0
    pmaddubsw            m1, m2
    pmaddubsw            m0, m2
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m5, m0
    paddw                m5, m1
    lea                  yq, [yq+strideq*4]
    add                 acq, 32
    sub                  hd, 4
    jg .w4_loop
    test              hpadd, hpadd
    jz .calc_avg_4
    punpckhqdq           m0, m0
.w4_hpad_loop:
    mova              [acq], m0
    paddw                m5, m0
    add                 acq, 16
    sub               hpadd, 2
    jg .w4_hpad_loop
.calc_avg_4:
    psrlw                m2, 10
    pmaddwd              m5, m2
    jmp .calc_avg

.w8:
    lea            stride3q, [strideq*3]
    test              wpadd, wpadd
    jnz .w8_wpad
.w8_loop:
    movq                 m1, [yq]
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    movq                 m0, [yq+strideq]
    punpcklbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0
    movq                 m1, [yq+strideq*2]
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    movq                 m0, [yq+stride3q]
    punpcklbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*4]
    add                 acq, 64
    sub                  hd, 4
    jg .w8_loop
    test              hpadd, hpadd
    jz .calc_avg_8_16
    jmp .w8_hpad
.w8_wpad:
    movd                 m1, [yq]
    punpcklbw            m1, m1
    punpcklqdq           m1, m1
    pmaddubsw            m1, m2
    pshufhw              m1, m1, q3333
    mova              [acq], m1
    paddw                m5, m1
    movd                 m0, [yq+strideq]
    punpcklbw            m0, m0
    punpcklqdq           m0, m0
    pmaddubsw            m0, m2
    pshufhw              m0, m0, q3333
    mova           [acq+16], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 32
    sub                  hd, 2
    jg .w8_wpad
    test              hpadd, hpadd
    jz .calc_avg_8_16
.w8_hpad:
    mova              [acq], m0
    paddw                m5, m0
    mova           [acq+16], m0
    paddw                m4, m0
    add                 acq, 32
    sub               hpadd, 2
    jg .w8_hpad
    jmp .calc_avg_8_16

.w16:
    test              wpadd, wpadd
    jnz .w16_wpad
.w16_loop:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0
    mova                 m0, [yq+strideq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_loop
    test              hpadd, hpadd
    jz .calc_avg_8_16
    jmp .w16_hpad_loop
.w16_wpad:
    cmp               wpadd, 2
    jl .w16_pad1
    je .w16_pad2
.w16_pad3:
    movd                 m1, [yq]
    punpcklbw            m1, m1
    punpcklqdq           m1, m1
    pshufhw              m1, m1, q3333
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    punpckhqdq           m1, m1
    mova           [acq+16], m1
    paddw                m5, m1
    movd                 m1, [yq+strideq]
    punpcklbw            m1, m1
    punpcklqdq           m1, m1
    pshufhw              m1, m1, q3333
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    punpckhqdq           m0, m1, m1
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad3
    jmp .w16_wpad_done
.w16_pad2:
    movq                 m1, [yq]
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    pshufhw              m1, m1, q3333
    punpckhqdq           m1, m1
    mova           [acq+16], m1
    paddw                m5, m1
    movq                 m1, [yq+strideq]
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    mova                 m0, m1
    pshufhw              m0, m0, q3333
    punpckhqdq           m0, m0
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad2
    jmp .w16_wpad_done
.w16_pad1:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1
    punpckhbw            m0, m0
    punpcklqdq           m0, m0
    pshufhw              m0, m0, q3333
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0
    mova                 m0, [yq+strideq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova           [acq+32], m1
    paddw                m4, m1
    punpckhbw            m0, m0
    punpcklqdq           m0, m0
    pshufhw              m0, m0, q3333
    pmaddubsw            m0, m2
    mova           [acq+48], m0
    paddw                m4, m0
    lea                  yq, [yq+strideq*2]
    add                 acq, 64
    sub                  hd, 2
    jg .w16_pad1
.w16_wpad_done:
    test              hpadd, hpadd
    jz .calc_avg_8_16
.w16_hpad_loop:
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m4, m1
    paddw                m5, m0
    mova           [acq+32], m1
    mova           [acq+48], m0
    paddw                m4, m1
    paddw                m5, m0
    add                 acq, 64
    sub               hpadd, 2
    jg .w16_hpad_loop
.calc_avg_8_16:
    mova                 m0, m5
    psrld                m5, 16
    pslld                m0, 16
    psrld                m0, 16
    paddd                m5, m0
    mova                 m0, m4
    psrld                m0, 16
    pslld                m4, 16
    psrld                m4, 16
    paddd                m0, m4
    paddd                m5, m0
    jmp .calc_avg

.w32:
    pxor                 m0, m0
    mova           [rsp   ], m0
    mova           [rsp+16], m0
    mova           [rsp+32], m0
    mova           [rsp+48], m0
    test              wpadd, wpadd
    jnz .w32_wpad
.w32_loop:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m4, [yq+16]
    mova                 m3, m4
    punpcklbw            m3, m3
    pmaddubsw            m3, m2
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    punpckhbw            m4, m4
    pmaddubsw            m4, m2
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_loop
    test              hpadd, hpadd
    jz .calc_avg_32
    jmp .w32_hpad_loop
.w32_wpad:
    cmp               wpadd, 2
    jl .w32_pad1
    je .w32_pad2
    cmp               wpadd, 4
    jl .w32_pad3
    je .w32_pad4
    cmp               wpadd, 6
    jl .w32_pad5
    je .w32_pad6
.w32_pad7:
    movd                 m1, [yq]
    punpcklbw            m1, m1
    punpcklqdq           m1, m1
    pshufhw              m1, m1, q3333
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    mova                 m0, m1
    punpckhqdq           m0, m0
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m3, m0
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    mova                 m4, m3
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad7
    jmp .w32_wpad_done
.w32_pad6:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    pshufhw              m0, m1, q3333
    punpckhqdq           m0, m0
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m3, m0
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    mova                 m4, m3
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad6
    jmp .w32_wpad_done
.w32_pad5:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    mova                 m5, [rsp]
    paddw                m5, m1
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    punpcklqdq           m0, m0
    pshufhw              m0, m0, q3333
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m3, m0
    punpckhqdq           m3, m3
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    mova                 m4, m3
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad5
    jmp .w32_wpad_done
.w32_pad4:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m3, m0
    pshufhw              m3, m3, q3333
    punpckhqdq           m3, m3
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    mova                 m4, m3
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad4
    jmp .w32_wpad_done
.w32_pad3:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    movd                 m3, [yq+16]
    punpcklbw            m3, m3
    punpcklqdq           m3, m3
    pshufhw              m3, m3, q3333
    pmaddubsw            m3, m2
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    mova                 m4, m3
    punpckhqdq           m4, m4
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad3
    jmp .w32_wpad_done
.w32_pad2:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m3, [yq+16]
    punpcklbw            m3, m3
    pmaddubsw            m3, m2
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    pshufhw              m4, m3, q3333
    punpckhqdq           m4, m4
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad2
    jmp .w32_wpad_done
.w32_pad1:
    mova                 m0, [yq]
    mova                 m1, m0
    punpcklbw            m1, m1
    pmaddubsw            m1, m2
    mova              [acq], m1
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    punpckhbw            m0, m0
    pmaddubsw            m0, m2
    mova           [acq+16], m0
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova                 m4, [yq+16]
    mova                 m3, m4
    punpcklbw            m3, m3
    pmaddubsw            m3, m2
    mova           [acq+32], m3
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    punpckhbw            m4, m4
    punpcklqdq           m4, m4
    pshufhw              m4, m4, q3333
    pmaddubsw            m4, m2
    mova           [acq+48], m4
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    lea                  yq, [yq+strideq]
    add                 acq, 64
    sub                  hd, 1
    jg .w32_pad1
.w32_wpad_done:
    test              hpadd, hpadd
    jz .calc_avg_32
.w32_hpad_loop:
    mova              [acq], m1
    mova           [acq+16], m0
    paddw                m5, m1, [rsp]
    mova           [rsp   ], m5
    paddw                m5, m0, [rsp+16]
    mova           [rsp+16], m5
    mova           [acq+32], m3
    mova           [acq+48], m4
    paddw                m5, m3, [rsp+32]
    mova           [rsp+32], m5
    paddw                m5, m4, [rsp+48]
    mova           [rsp+48], m5
    add                 acq, 64
    sub               hpadd, 1
    jg .w32_hpad_loop

%if ARCH_X86_64
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h, ac_bak
%else
    DEFINE_ARGS ac, y, iptr, wpad, hpad, sz, h
%endif

.calc_avg_32:
    mova                 m5, [rsp]
    mova                 m0, m5
    psrld                m5, 16
    pslld                m0, 16
    psrld                m0, 16
    paddd                m5, m0
    mova                 m0, [rsp+16]
    mova                 m3, m0
    psrld                m0, 16
    pslld                m3, 16
    psrld                m3, 16
    paddd                m0, m3
    paddd                m5, m0
    mova                 m0, [rsp+32]
    mova                 m3, m0
    psrld                m0, 16
    pslld                m3, 16
    psrld                m3, 16
    paddd                m0, m3
    mova                 m1, [rsp+48]
    mova                 m3, m1
    psrld                m1, 16
    pslld                m3, 16
    psrld                m3, 16
    paddd                m1, m3
    paddd                m1, m0
    paddd                m5, m1
.calc_avg:
    movd                szd, m6
    psrad                m6, 1
    tzcnt               r1d, szd                       ; const int log2sz = ctz(width) + ctz(height);
    paddd                m5, m6
    movd                 m1, r1d
    pshufd               m0, m5, q2301
    paddd                m0, m5
    pshufd               m5, m0, q1032
    paddd                m0, m5
    psrad                m0, m1                        ; sum >>= log2sz;
    packssdw             m0, m0
    RELOAD_ACQ_32       acq                            ; ac = ac_orig
.sub_loop:
    mova                 m1, [acq]
    psubw                m1, m0
    mova              [acq], m1
    add                 acq, 16
    sub                 szd, 8
    jg .sub_loop
    RET

; %1 simd register that hold the mask and will hold the result
; %2 simd register that holds the "true" values
; %3 location of the "false" values (simd register/memory)
%macro BLEND 3 ; mask, true, false
    pand  %2, %1
    pandn %1, %3
    por   %1, %2
%endmacro

%macro PAETH 2                                 ; top, ldiff
    pavgb                m1, m%1, m3
    pxor                 m0, m%1, m3
    pand                 m0, m4
    psubusb              m2, m5, m1
    psubb                m1, m0
    psubusb              m1, m5
    por                  m1, m2
    paddusb              m1, m1
    por                  m1, m0               ; min(tldiff, 255)
    psubusb              m2, m5, m3
    psubusb              m0, m3, m5
    por                  m2, m0               ; tdiff
%ifnum %2
    pminub               m2, m%2
    pcmpeqb              m0, m%2, m2          ; ldiff <= tdiff
%else
    mova                 m0, %2
    pminub               m2, m0
    pcmpeqb              m0, m2
%endif
    pminub               m1, m2
    pcmpeqb              m1, m2               ; ldiff <= tldiff && tdiff <= tldiff
    mova                 m2, m3
    BLEND                m0, m2, m%1
    BLEND                m1, m0, m5
%endmacro

cglobal ipred_paeth_8bpc, 3, 6, 8, -7*16, dst, stride, tl, w, h
%define base r5-ipred_paeth_ssse3_table
    tzcnt                wd, wm
    movifnidn            hd, hm
    pxor                 m0, m0
    movd                 m5, [tlq]
    pshufb               m5, m0
    LEA                  r5, ipred_paeth_ssse3_table
    movsxd               wq, [r5+wq*4]
    movddup              m4, [base+ipred_paeth_shuf]
    add                  wq, r5
    jmp                  wq
.w4:
    movd                 m6, [tlq+1]            ; top
    pshufd               m6, m6, q0000
    lea                  r3, [strideq*3]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0                 ; ldiff
.w4_loop:
    sub                 tlq, 4
    movd                 m3, [tlq]
    mova                 m1, [base+ipred_h_shuf]
    pshufb               m3, m1                 ; left
    PAETH                 6, 7
    movd   [dstq          ], m1
    pshuflw              m0, m1, q1032
    movd   [dstq+strideq  ], m0
    punpckhqdq           m1, m1
    movd   [dstq+strideq*2], m1
    psrlq                m1, 32
    movd   [dstq+r3       ], m1
    lea                dstq, [dstq+strideq*4]
    sub                  hd, 4
    jg .w4_loop
    RET
ALIGN function_align
.w8:
    movddup              m6, [tlq+1]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
.w8_loop:
    sub                 tlq, 2
    movd                 m3, [tlq]
    pshufb               m3, [base+ipred_paeth_shuf]
    PAETH                 6, 7
    movq     [dstq        ], m1
    movhps   [dstq+strideq], m1
    lea                dstq, [dstq+strideq*2]
    sub                  hd, 2
    jg .w8_loop
    RET
ALIGN function_align
.w16:
    movu                 m6, [tlq+1]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
.w16_loop:
    sub                 tlq, 1
    movd                 m3, [tlq]
    pxor                 m1, m1
    pshufb               m3, m1
    PAETH                 6, 7
    mova             [dstq], m1
    add                dstq, strideq
    sub                  hd, 1
    jg .w16_loop
    RET
ALIGN function_align
.w32:
    movu                 m6, [tlq+1]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp   ], m6
    mova           [rsp+16], m7
    movu                 m6, [tlq+17]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp+32], m6
.w32_loop:
    dec                 tlq
    movd                 m3, [tlq]
    pxor                 m1, m1
    pshufb               m3, m1
    mova                 m6, [rsp]
    PAETH                 6, [rsp+16]
    mova          [dstq   ], m1
    mova                 m6, [rsp+32]
    PAETH                 6, 7
    mova          [dstq+16], m1
    add                dstq, strideq
    dec                  hd
    jg .w32_loop
    RET
ALIGN function_align
.w64:
    movu                 m6, [tlq+1]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp   ], m6
    mova           [rsp+16], m7
    movu                 m6, [tlq+17]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp+32], m6
    mova           [rsp+48], m7
    movu                 m6, [tlq+33]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp+64], m6
    mova           [rsp+80], m7
    movu                 m6, [tlq+49]
    psubusb              m7, m5, m6
    psubusb              m0, m6, m5
    por                  m7, m0
    mova           [rsp+96], m6
.w64_loop:
    dec                 tlq
    movd                 m3, [tlq]
    pxor                 m1, m1
    pshufb               m3, m1
    mova                 m6, [rsp]
    PAETH                 6, [rsp+16]
    mova          [dstq   ], m1
    mova                 m6, [rsp+32]
    PAETH                 6, [rsp+48]
    mova          [dstq+16], m1
    mova                 m6, [rsp+64]
    PAETH                 6, [rsp+80]
    mova          [dstq+32], m1
    mova                 m6, [rsp+96]
    PAETH                 6, 7
    mova          [dstq+48], m1
    add                dstq, strideq
    dec                  hd
    jg .w64_loop
    RET


%macro FILTER 4  ;dst, src, tmp, shuf
%ifnum %4
    pshufb               m%2, m%4
%else
    pshufb               m%2, %4
%endif
    pshufd               m%1, m%2, q0000           ;p0 p1
    pmaddubsw            m%1, m2
    pshufd               m%3, m%2, q1111           ;p2 p3
    pmaddubsw            m%3, m3
    paddw                m%1, [base+pw_8]
    paddw                m%1, m%3
    pshufd               m%3, m%2, q2222           ;p4 p5
    pmaddubsw            m%3, m4
    paddw                m%1, m%3
    pshufd               m%3, m%2, q3333           ;p6 __
    pmaddubsw            m%3, m5
    paddw                m%1, m%3
    psraw                m%1, 4
    packuswb             m%1, m%1
%endmacro

cglobal ipred_filter_8bpc, 3, 7, 8, dst, stride, tl, w, h, filter
%define base r6-$$
    LEA                   r6, $$
    tzcnt                 wd, wm
%ifidn filterd, filterm
    movzx            filterd, filterb
%else
    movzx            filterd, byte filterm
%endif
    shl              filterd, 6
    lea              filterq, [base+filter_intra_taps+filterq]
    movq                  m0, [tlq-3]                     ;_ 6 5 0 1 2 3 4
    movsxd                wq, [base+ipred_filter_ssse3_table+wq*4]
    mova                  m2, [filterq+16*0]
    mova                  m3, [filterq+16*1]
    mova                  m4, [filterq+16*2]
    mova                  m5, [filterq+16*3]
    lea                   wq, [base+ipred_filter_ssse3_table+wq]
    mov                   hd, hm
    jmp                   wq
.w4:
    mova                  m1, [base+filter_shuf1]
    sub                  tlq, 3
    sub                  tlq, hq
    jmp .w4_loop_start
.w4_loop:
    movd                  m0, [tlq+hq]
    punpckldq             m0, m6
    lea                 dstq, [dstq+strideq*2]
.w4_loop_start:
    FILTER                 6, 0, 7, 1
    movd    [dstq+strideq*0], m6
    pshuflw               m6, m6, q1032
    movd    [dstq+strideq*1], m6
    sub                   hd, 2
    jg .w4_loop
    RET

ALIGN function_align
.w8:
    movq                  m6, [tlq+1]                   ;_ _ _ 0 1 2 3 4
    sub                  tlq, 5
    sub                  tlq, hq

.w8_loop:
    FILTER                 7, 0, 1, [base+filter_shuf1]
    punpcklqdq            m6, m7                        ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    FILTER                 0, 6, 1, [base+filter_shuf2]

    punpckldq             m6, m7, m0
    movq    [dstq+strideq*0], m6
    punpckhqdq            m6, m6
    movq    [dstq+strideq*1], m6

    movd                  m0, [tlq+hq]                  ;_ 6 5 0
    punpckldq             m0, m6                        ;_ 6 5 0 1 2 3 4

    lea                 dstq, [dstq+strideq*2]
    sub                   hd, 2
    jg .w8_loop
    RET

ALIGN function_align
.w16:
    movu                  m6, [tlq+1]                   ;top row
    sub                  tlq, 5
    sub                  tlq, hq

.w16_loop:
    FILTER                 7, 0, 1, [base+filter_shuf1]
    punpcklqdq            m0, m6, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd    [dstq+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m6, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m7, m6                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd  [dstq+4+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4

    FILTER                 7, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m6, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd  [dstq+8+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m6, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    movd [dstq+12+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4
    mova    [dstq+strideq*1], m6

    movd                  m0, [tlq+hq]                  ;_ 6 5 0
    punpckldq             m0, m6                        ;_ 6 5 0 1 2 3 4

    lea                 dstq, [dstq+strideq*2]
    sub                   hd, 2
    jg .w16_loop
    RET

ALIGN function_align
.w32:
    movu                  m6, [tlq+1]                   ;top row
    lea              filterq, [tlq+17]
    sub                  tlq, 5
    sub                  tlq, hq

.w32_loop:
    FILTER                 7, 0, 1, [base+filter_shuf1]
    punpcklqdq            m0, m6, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd    [dstq+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m6, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m7, m6                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd  [dstq+4+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4

    FILTER                 7, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m6, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd  [dstq+8+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m6, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    movu                  m1, [filterq]
    punpckldq             m0, m7, m1                    ;_ _ _ 0 1 2 3 4 _ _ _ _ _ _ _ _
    punpcklqdq            m0, m6                        ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd [dstq+12+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4
    mova    [dstq+strideq*1], m6

    mova                  m6, m1

    FILTER                 7, 0, 6, [base+filter_shuf2]
    punpcklqdq            m0, m1, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd [dstq+16+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m1, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m7, m6                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd [dstq+20+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4

    FILTER                 7, 0, 1, [base+filter_shuf2]
    punpcklqdq            m0, m6, m7                    ;_ _ _ 0 1 2 3 4 _ _ _ 5 _ _ _ 6
    movd [dstq+24+strideq*0], m7
    psrlq                 m7, 32
    palignr               m7, m6, 4

    FILTER                 6, 0, 1, [base+filter_shuf2]
    movd [dstq+28+strideq*0], m6
    psrlq                 m6, 32
    palignr               m6, m7, 4
    mova [dstq+16+strideq*1], m6

    mova                  m6, [dstq+strideq*1]
    movd                  m0, [tlq+hq]                  ;_ 6 5 0
    punpckldq             m0, m6                        ;_ 6 5 0 1 2 3 4
    lea              filterq, [dstq+16+strideq*1]
    lea                 dstq, [dstq+strideq*2]
    sub                   hd, 2
    jg .w32_loop
    RET