/*
 * Copyright © 2018, VideoLAN and dav1d authors
 * Copyright © 2019, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

// void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int a,
//                        const int max_width, const int max_height);
function ipred_dc_128_neon, export=1
        clz             w3,  w3
        adr             x5,  L(ipred_dc_128_tbl)
        sub             w3,  w3,  #25
        ldrh            w3,  [x5, w3, uxtw #1]
        movi            v0.16b,  #128
        sub             x5,  x5,  w3, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
4:
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            4b
        ret
8:
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            8b
        ret
16:
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            16b
        ret
320:
        movi            v1.16b,  #128
32:
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        b.gt            32b
        ret
640:
        movi            v1.16b,  #128
        movi            v2.16b,  #128
        movi            v3.16b,  #128
64:
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        b.gt            64b
        ret

L(ipred_dc_128_tbl):
        .hword L(ipred_dc_128_tbl) - 640b
        .hword L(ipred_dc_128_tbl) - 320b
        .hword L(ipred_dc_128_tbl) -  16b
        .hword L(ipred_dc_128_tbl) -   8b
        .hword L(ipred_dc_128_tbl) -   4b
endfunc

// void ipred_v_neon(pixel *dst, const ptrdiff_t stride,
//                   const pixel *const topleft,
//                   const int width, const int height, const int a,
//                   const int max_width, const int max_height);
function ipred_v_neon, export=1
        clz             w3,  w3
        adr             x5,  L(ipred_v_tbl)
        sub             w3,  w3,  #25
        ldrh            w3,  [x5, w3, uxtw #1]
        add             x2,  x2,  #1
        sub             x5,  x5,  w3, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        ld1             {v0.s}[0],  [x2]
4:
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            4b
        ret
80:
        ld1             {v0.8b},  [x2]
8:
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            8b
        ret
160:
        ld1             {v0.16b}, [x2], #16
16:
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            16b
        ret
320:
        ld1             {v0.16b, v1.16b}, [x2]
32:
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        b.gt            32b
        ret
640:
        ld1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x2]
64:
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        b.gt            64b
        ret

L(ipred_v_tbl):
        .hword L(ipred_v_tbl) - 640b
        .hword L(ipred_v_tbl) - 320b
        .hword L(ipred_v_tbl) - 160b
        .hword L(ipred_v_tbl) -  80b
        .hword L(ipred_v_tbl) -  40b
endfunc

// void ipred_h_neon(pixel *dst, const ptrdiff_t stride,
//                   const pixel *const topleft,
//                   const int width, const int height, const int a,
//                   const int max_width, const int max_height);
function ipred_h_neon, export=1
        clz             w3,  w3
        adr             x5,  L(ipred_h_tbl)
        sub             w3,  w3,  #25
        ldrh            w3,  [x5, w3, uxtw #1]
        sub             x2,  x2,  #4
        sub             x5,  x5,  w3, uxtw
        mov             x7,  #-4
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
4:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7
        st1             {v3.s}[0],  [x0], x1
        st1             {v2.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v1.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            4b
        ret
8:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7
        st1             {v3.8b},  [x0], x1
        st1             {v2.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v1.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            8b
        ret
16:
        ld4r            {v0.16b, v1.16b, v2.16b, v3.16b},  [x2], x7
        st1             {v3.16b}, [x0], x1
        st1             {v2.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v1.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            16b
        ret
32:
        ld4r            {v0.16b, v1.16b, v2.16b, v3.16b},  [x2], x7
        str             q3,  [x0, #16]
        str             q2,  [x6, #16]
        st1             {v3.16b}, [x0], x1
        st1             {v2.16b}, [x6], x1
        subs            w4,  w4,  #4
        str             q1,  [x0, #16]
        str             q0,  [x6, #16]
        st1             {v1.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            32b
        ret
64:
        ld4r            {v0.16b, v1.16b, v2.16b, v3.16b},  [x2], x7
        str             q3,  [x0, #16]
        str             q2,  [x6, #16]
        stp             q3,  q3,  [x0, #32]
        stp             q2,  q2,  [x6, #32]
        st1             {v3.16b}, [x0], x1
        st1             {v2.16b}, [x6], x1
        subs            w4,  w4,  #4
        str             q1,  [x0, #16]
        str             q0,  [x6, #16]
        stp             q1,  q1,  [x0, #32]
        stp             q0,  q0,  [x6, #32]
        st1             {v1.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            64b
        ret

L(ipred_h_tbl):
        .hword L(ipred_h_tbl) - 64b
        .hword L(ipred_h_tbl) - 32b
        .hword L(ipred_h_tbl) - 16b
        .hword L(ipred_h_tbl) -  8b
        .hword L(ipred_h_tbl) -  4b
endfunc

// void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int a,
//                        const int max_width, const int max_height);
function ipred_dc_top_neon, export=1
        clz             w3,  w3
        adr             x5,  L(ipred_dc_top_tbl)
        sub             w3,  w3,  #25
        ldrh            w3,  [x5, w3, uxtw #1]
        add             x2,  x2,  #1
        sub             x5,  x5,  w3, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        ld1r            {v0.2s},  [x2]
        uaddlv          h0,      v0.8b
        rshrn           v0.8b,   v0.8h,   #3
        dup             v0.8b,   v0.b[0]
4:
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            4b
        ret
80:
        ld1             {v0.8b},  [x2]
        uaddlv          h0,      v0.8b
        rshrn           v0.8b,   v0.8h,   #3
        dup             v0.8b,   v0.b[0]
8:
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            8b
        ret
160:
        ld1             {v0.16b}, [x2]
        uaddlv          h0,      v0.16b
        rshrn           v0.8b,   v0.8h,   #4
        dup             v0.16b,  v0.b[0]
16:
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            16b
        ret
320:
        ld1             {v0.16b, v1.16b}, [x2]
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        add             v2.4h,   v0.4h,   v1.4h
        rshrn           v2.8b,   v2.8h,   #5
        dup             v0.16b,  v2.b[0]
        dup             v1.16b,  v2.b[0]
32:
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        b.gt            32b
        ret
640:
        ld1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x2]
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v4.4h,   v0.4h,   v1.4h
        add             v5.4h,   v2.4h,   v3.4h
        add             v4.4h,   v4.4h,   v5.4h
        rshrn           v4.8b,   v4.8h,   #6
        dup             v0.16b,  v4.b[0]
        dup             v1.16b,  v4.b[0]
        dup             v2.16b,  v4.b[0]
        dup             v3.16b,  v4.b[0]
64:
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        b.gt            64b
        ret

L(ipred_dc_top_tbl):
        .hword L(ipred_dc_top_tbl) - 640b
        .hword L(ipred_dc_top_tbl) - 320b
        .hword L(ipred_dc_top_tbl) - 160b
        .hword L(ipred_dc_top_tbl) -  80b
        .hword L(ipred_dc_top_tbl) -  40b
endfunc

// void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride,
//                         const pixel *const topleft,
//                         const int width, const int height, const int a,
//                         const int max_width, const int max_height);
function ipred_dc_left_neon, export=1
        sub             x2,  x2,  w4, uxtw
        clz             w3,  w3
        clz             w7,  w4
        adr             x5,  L(ipred_dc_left_tbl)
        sub             w3,  w3,  #20 // 25 leading bits, minus table offset 5
        sub             w7,  w7,  #25
        ldrh            w3,  [x5, w3, uxtw #1]
        ldrh            w7,  [x5, w7, uxtw #1]
        sub             x3,  x5,  w3, uxtw
        sub             x5,  x5,  w7, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5

L(ipred_dc_left_h4):
        ld1r            {v0.2s},  [x2]
        uaddlv          h0,      v0.8b
        rshrn           v0.8b,   v0.8h,   #3
        dup             v0.16b,  v0.b[0]
        br              x3
L(ipred_dc_left_w4):
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            L(ipred_dc_left_w4)
        ret

L(ipred_dc_left_h8):
        ld1             {v0.8b},  [x2]
        uaddlv          h0,      v0.8b
        rshrn           v0.8b,   v0.8h,   #3
        dup             v0.16b,  v0.b[0]
        br              x3
L(ipred_dc_left_w8):
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            L(ipred_dc_left_w8)
        ret

L(ipred_dc_left_h16):
        ld1             {v0.16b}, [x2]
        uaddlv          h0,      v0.16b
        rshrn           v0.8b,   v0.8h,   #4
        dup             v0.16b,  v0.b[0]
        br              x3
L(ipred_dc_left_w16):
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            L(ipred_dc_left_w16)
        ret

L(ipred_dc_left_h32):
        ld1             {v0.16b, v1.16b}, [x2]
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        add             v0.4h,   v0.4h,   v1.4h
        rshrn           v0.8b,   v0.8h,   #5
        dup             v0.16b,  v0.b[0]
        br              x3
L(ipred_dc_left_w32):
        mov             v1.16b,  v0.16b
1:
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        b.gt            1b
        ret

L(ipred_dc_left_h64):
        ld1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x2]
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v0.4h,   v0.4h,   v1.4h
        add             v2.4h,   v2.4h,   v3.4h
        add             v0.4h,   v0.4h,   v2.4h
        rshrn           v0.8b,   v0.8h,   #6
        dup             v0.16b,  v0.b[0]
        br              x3
L(ipred_dc_left_w64):
        mov             v1.16b,  v0.16b
        mov             v2.16b,  v0.16b
        mov             v3.16b,  v0.16b
1:
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        b.gt            1b
        ret

L(ipred_dc_left_tbl):
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h64)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h32)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h16)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h8)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h4)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w64)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w32)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8)
        .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4)
endfunc

// void ipred_dc_neon(pixel *dst, const ptrdiff_t stride,
//                    const pixel *const topleft,
//                    const int width, const int height, const int a,
//                    const int max_width, const int max_height);
function ipred_dc_neon, export=1
        sub             x2,  x2,  w4, uxtw
        add             w7,  w3,  w4             // width + height
        clz             w3,  w3
        clz             w6,  w4
        dup             v16.8h, w7               // width + height
        adr             x5,  L(ipred_dc_tbl)
        rbit            w7,  w7                  // rbit(width + height)
        sub             w3,  w3,  #20            // 25 leading bits, minus table offset 5
        sub             w6,  w6,  #25
        clz             w7,  w7                  // ctz(width + height)
        ldrh            w3,  [x5, w3, uxtw #1]
        ldrh            w6,  [x5, w6, uxtw #1]
        neg             w7,  w7                  // -ctz(width + height)
        sub             x3,  x5,  w3, uxtw
        sub             x5,  x5,  w6, uxtw
        ushr            v16.8h,  v16.8h,  #1     // (width + height) >> 1
        dup             v17.8h,  w7              // -ctz(width + height)
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5

L(ipred_dc_h4):
        ld1             {v0.s}[0],  [x2], #4
        ins             v0.s[1], wzr
        uaddlv          h0,      v0.8b
        br              x3
L(ipred_dc_w4):
        add             x2,  x2,  #1
        ld1             {v1.s}[0],  [x2]
        ins             v1.s[1], wzr
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h1,      v1.8b
        cmp             w4,  #4
        add             v0.4h,   v0.4h,   v1.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 8/16
        mov             w16, #(0x3334/2)
        movk            w16, #(0x5556/2), lsl #16
        add             w17, w4,  w4  // w17 = 2*h = 16 or 32
        lsr             w16, w16, w17
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8b,   v0.b[0]
2:
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.s}[0],  [x0], x1
        st1             {v0.s}[0],  [x6], x1
        b.gt            2b
        ret

L(ipred_dc_h8):
        ld1             {v0.8b},  [x2], #8
        uaddlv          h0,      v0.8b
        br              x3
L(ipred_dc_w8):
        add             x2,  x2,  #1
        ld1             {v1.8b},  [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h1,      v1.8b
        cmp             w4,  #8
        add             v0.4h,   v0.4h,   v1.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 4/16/32
        cmp             w4,  #32
        mov             w16, #(0x3334/2)
        mov             w17, #(0x5556/2)
        csel            w16, w16, w17, eq
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8b,   v0.b[0]
2:
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.8b},  [x0], x1
        st1             {v0.8b},  [x6], x1
        b.gt            2b
        ret

L(ipred_dc_h16):
        ld1             {v0.16b}, [x2], #16
        uaddlv          h0,      v0.16b
        br              x3
L(ipred_dc_w16):
        add             x2,  x2,  #1
        ld1             {v1.16b}, [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h1,      v1.16b
        cmp             w4,  #16
        add             v0.4h,   v0.4h,   v1.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 4/8/32/64
        tst             w4,  #(32+16+8) // 16 added to make a consecutive bitmask
        mov             w16, #(0x3334/2)
        mov             w17, #(0x5556/2)
        csel            w16, w16, w17, eq
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.16b,  v0.b[0]
2:
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b}, [x0], x1
        st1             {v0.16b}, [x6], x1
        b.gt            2b
        ret

L(ipred_dc_h32):
        ld1             {v0.16b, v1.16b}, [x2], #32
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        add             v0.4h,   v0.4h,   v1.4h
        br              x3
L(ipred_dc_w32):
        add             x2,  x2,  #1
        ld1             {v1.16b, v2.16b}, [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h1,      v1.16b
        uaddlv          h2,      v2.16b
        cmp             w4,  #32
        add             v0.4h,   v0.4h,   v1.4h
        add             v0.4h,   v0.4h,   v2.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 8/16/64
        cmp             w4,  #8
        mov             w16, #(0x3334/2)
        mov             w17, #(0x5556/2)
        csel            w16, w16, w17, eq
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.16b,  v0.b[0]
        dup             v1.16b,  v0.b[0]
2:
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b}, [x0], x1
        st1             {v0.16b, v1.16b}, [x6], x1
        b.gt            2b
        ret

L(ipred_dc_h64):
        ld1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #64
        uaddlv          h0,      v0.16b
        uaddlv          h1,      v1.16b
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v0.4h,   v0.4h,   v1.4h
        add             v2.4h,   v2.4h,   v3.4h
        add             v0.4h,   v0.4h,   v2.4h
        br              x3
L(ipred_dc_w64):
        mov             v1.16b,  v0.16b
        mov             v2.16b,  v0.16b
        mov             v3.16b,  v0.16b
2:
        add             x2,  x2,  #1
        ld1             {v1.16b, v2.16b, v3.16b, v4.16b}, [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h1,      v1.16b
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        uaddlv          h4,      v4.16b
        add             v1.4h,   v1.4h,   v2.4h
        add             v3.4h,   v3.4h,   v4.4h
        cmp             w4,  #64
        add             v0.4h,   v0.4h,   v1.4h
        add             v0.4h,   v0.4h,   v3.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 16/32
        mov             w16, #(0x5556/2)
        movk            w16, #(0x3334/2), lsl #16
        lsr             w16, w16, w4
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.16b,  v0.b[0]
        dup             v1.16b,  v0.b[0]
        dup             v2.16b,  v0.b[0]
        dup             v3.16b,  v0.b[0]
2:
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1
        st1             {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1
        b.gt            2b
        ret

L(ipred_dc_tbl):
        .hword L(ipred_dc_tbl) - L(ipred_dc_h64)
        .hword L(ipred_dc_tbl) - L(ipred_dc_h32)
        .hword L(ipred_dc_tbl) - L(ipred_dc_h16)
        .hword L(ipred_dc_tbl) - L(ipred_dc_h8)
        .hword L(ipred_dc_tbl) - L(ipred_dc_h4)
        .hword L(ipred_dc_tbl) - L(ipred_dc_w64)
        .hword L(ipred_dc_tbl) - L(ipred_dc_w32)
        .hword L(ipred_dc_tbl) - L(ipred_dc_w16)
        .hword L(ipred_dc_tbl) - L(ipred_dc_w8)
        .hword L(ipred_dc_tbl) - L(ipred_dc_w4)
endfunc

// void ipred_paeth_neon(pixel *dst, const ptrdiff_t stride,
//                       const pixel *const topleft,
//                       const int width, const int height, const int a,
//                       const int max_width, const int max_height);
function ipred_paeth_neon, export=1
        clz             w9,  w3
        adr             x5,  L(ipred_paeth_tbl)
        sub             w9,  w9,  #25
        ldrh            w9,  [x5, w9, uxtw #1]
        ld1r            {v4.16b},  [x2]
        add             x8,  x2,  #1
        sub             x2,  x2,  #4
        sub             x5,  x5,  w9, uxtw
        mov             x7,  #-4
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        ld1r            {v5.4s},  [x8]
        usubl           v6.8h,   v5.8b,   v4.8b   // top - topleft
4:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7
        zip1            v0.2s,   v0.2s,   v1.2s
        zip1            v2.2s,   v2.2s,   v3.2s
        uaddw           v16.8h,  v6.8h,   v0.8b
        uaddw           v17.8h,  v6.8h,   v2.8b
        sqxtun          v16.8b,  v16.8h           // base
        sqxtun2         v16.16b, v17.8h
        zip1            v0.2d,   v0.2d,   v2.2d
        uabd            v20.16b, v5.16b,  v16.16b // tdiff
        uabd            v22.16b, v4.16b,  v16.16b // tldiff
        uabd            v16.16b, v0.16b,  v16.16b // ldiff
        umin            v18.16b, v20.16b, v22.16b // min(tdiff, tldiff)
        cmhs            v20.16b, v22.16b, v20.16b // tldiff >= tdiff
        cmhs            v16.16b, v18.16b, v16.16b // min(tdiff, tldiff) >= ldiff
        bsl             v20.16b, v5.16b,  v4.16b  // tdiff <= tldiff ? top : topleft
        bit             v20.16b, v0.16b,  v16.16b // ldiff <= min ? left : ...
        st1             {v20.s}[3], [x0], x1
        st1             {v20.s}[2], [x6], x1
        subs            w4,  w4,  #4
        st1             {v20.s}[1], [x0], x1
        st1             {v20.s}[0], [x6], x1
        b.gt            4b
        ret
80:
        ld1r            {v5.2d},  [x8]
        usubl           v6.8h,   v5.8b,   v4.8b   // top - topleft
8:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7
        uaddw           v16.8h,  v6.8h,   v0.8b
        uaddw           v17.8h,  v6.8h,   v1.8b
        uaddw           v18.8h,  v6.8h,   v2.8b
        uaddw           v19.8h,  v6.8h,   v3.8b
        sqxtun          v16.8b,  v16.8h           // base
        sqxtun2         v16.16b, v17.8h
        sqxtun          v18.8b,  v18.8h
        sqxtun2         v18.16b, v19.8h
        zip1            v2.2d,   v2.2d,   v3.2d
        zip1            v0.2d,   v0.2d,   v1.2d
        uabd            v21.16b, v5.16b,  v18.16b // tdiff
        uabd            v20.16b, v5.16b,  v16.16b
        uabd            v23.16b, v4.16b,  v18.16b // tldiff
        uabd            v22.16b, v4.16b,  v16.16b
        uabd            v17.16b, v2.16b,  v18.16b // ldiff
        uabd            v16.16b, v0.16b,  v16.16b
        umin            v19.16b, v21.16b, v23.16b // min(tdiff, tldiff)
        umin            v18.16b, v20.16b, v22.16b
        cmhs            v21.16b, v23.16b, v21.16b // tldiff >= tdiff
        cmhs            v20.16b, v22.16b, v20.16b
        cmhs            v17.16b, v19.16b, v17.16b // min(tdiff, tldiff) >= ldiff
        cmhs            v16.16b, v18.16b, v16.16b
        bsl             v21.16b, v5.16b,  v4.16b  // tdiff <= tldiff ? top : topleft
        bsl             v20.16b, v5.16b,  v4.16b
        bit             v21.16b, v2.16b,  v17.16b // ldiff <= min ? left : ...
        bit             v20.16b, v0.16b,  v16.16b
        st1             {v21.d}[1], [x0], x1
        st1             {v21.d}[0], [x6], x1
        subs            w4,  w4,  #4
        st1             {v20.d}[1], [x0], x1
        st1             {v20.d}[0], [x6], x1
        b.gt            8b
        ret
160:
320:
640:
        ld1             {v5.16b},  [x8], #16
        mov             w9,  w3
        // Set up pointers for four rows in parallel; x0, x6, x5, x10
        add             x5,  x0,  x1
        add             x10, x6,  x1
        lsl             x1,  x1,  #1
        sub             x1,  x1,  w3, uxtw
1:
        ld4r            {v0.16b, v1.16b, v2.16b, v3.16b},  [x2], x7
2:
        usubl           v6.8h,   v5.8b,   v4.8b   // top - topleft
        usubl2          v7.8h,   v5.16b,  v4.16b
        uaddw           v24.8h,  v6.8h,   v0.8b
        uaddw           v25.8h,  v7.8h,   v0.8b
        uaddw           v26.8h,  v6.8h,   v1.8b
        uaddw           v27.8h,  v7.8h,   v1.8b
        uaddw           v28.8h,  v6.8h,   v2.8b
        uaddw           v29.8h,  v7.8h,   v2.8b
        uaddw           v30.8h,  v6.8h,   v3.8b
        uaddw           v31.8h,  v7.8h,   v3.8b
        sqxtun          v17.8b,  v26.8h           // base
        sqxtun2         v17.16b, v27.8h
        sqxtun          v16.8b,  v24.8h
        sqxtun2         v16.16b, v25.8h
        sqxtun          v19.8b,  v30.8h
        sqxtun2         v19.16b, v31.8h
        sqxtun          v18.8b,  v28.8h
        sqxtun2         v18.16b, v29.8h
        uabd            v23.16b, v5.16b,  v19.16b // tdiff
        uabd            v22.16b, v5.16b,  v18.16b
        uabd            v21.16b, v5.16b,  v17.16b
        uabd            v20.16b, v5.16b,  v16.16b
        uabd            v27.16b, v4.16b,  v19.16b // tldiff
        uabd            v26.16b, v4.16b,  v18.16b
        uabd            v25.16b, v4.16b,  v17.16b
        uabd            v24.16b, v4.16b,  v16.16b
        uabd            v19.16b, v3.16b,  v19.16b // ldiff
        uabd            v18.16b, v2.16b,  v18.16b
        uabd            v17.16b, v1.16b,  v17.16b
        uabd            v16.16b, v0.16b,  v16.16b
        umin            v31.16b, v23.16b, v27.16b // min(tdiff, tldiff)
        umin            v30.16b, v22.16b, v26.16b
        umin            v29.16b, v21.16b, v25.16b
        umin            v28.16b, v20.16b, v24.16b
        cmhs            v23.16b, v27.16b, v23.16b // tldiff >= tdiff
        cmhs            v22.16b, v26.16b, v22.16b
        cmhs            v21.16b, v25.16b, v21.16b
        cmhs            v20.16b, v24.16b, v20.16b
        cmhs            v19.16b, v31.16b, v19.16b // min(tdiff, tldiff) >= ldiff
        cmhs            v18.16b, v30.16b, v18.16b
        cmhs            v17.16b, v29.16b, v17.16b
        cmhs            v16.16b, v28.16b, v16.16b
        bsl             v23.16b, v5.16b,  v4.16b  // tdiff <= tldiff ? top : topleft
        bsl             v22.16b, v5.16b,  v4.16b
        bsl             v21.16b, v5.16b,  v4.16b
        bsl             v20.16b, v5.16b,  v4.16b
        bit             v23.16b, v3.16b,  v19.16b // ldiff <= min ? left : ...
        bit             v22.16b, v2.16b,  v18.16b
        bit             v21.16b, v1.16b,  v17.16b
        bit             v20.16b, v0.16b,  v16.16b
        subs            w3,  w3,  #16
        st1             {v23.16b}, [x0],  #16
        st1             {v22.16b}, [x6],  #16
        st1             {v21.16b}, [x5],  #16
        st1             {v20.16b}, [x10], #16
        b.le            8f
        ld1             {v5.16b},  [x8], #16
        b               2b
8:
        subs            w4,  w4,  #4
        b.le            9f
        // End of horizontal loop, move pointers to next four rows
        sub             x8,  x8,  w9, uxtw
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        // Load the top row as early as possible
        ld1             {v5.16b},  [x8], #16
        add             x5,  x5,  x1
        add             x10, x10, x1
        mov             w3,  w9
        b               1b
9:
        ret

L(ipred_paeth_tbl):
        .hword L(ipred_paeth_tbl) - 640b
        .hword L(ipred_paeth_tbl) - 320b
        .hword L(ipred_paeth_tbl) - 160b
        .hword L(ipred_paeth_tbl) -  80b
        .hword L(ipred_paeth_tbl) -  40b
endfunc

// void ipred_smooth_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int a,
//                        const int max_width, const int max_height);
function ipred_smooth_neon, export=1
        movrel          x10, X(sm_weights)
        add             x11, x10, w4, uxtw
        add             x10, x10, w3, uxtw
        clz             w9,  w3
        adr             x5,  L(ipred_smooth_tbl)
        sub             x12, x2,  w4, uxtw
        sub             w9,  w9,  #25
        ldrh            w9,  [x5, w9, uxtw #1]
        ld1r            {v4.16b},  [x12] // bottom
        add             x8,  x2,  #1
        sub             x5,  x5,  w9, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        sub             x2,  x2,  #4
        mov             x7,  #-4
        ld1r            {v6.2s}, [x8]             // top
        ld1r            {v7.2s}, [x10]            // weights_hor
        dup             v5.16b,  v6.b[3]          // right
        usubl           v6.8h,   v6.8b,   v4.8b   // top-bottom
        uxtl            v7.8h,   v7.8b            // weights_hor
4:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7 // left
        ld4r            {v16.8b, v17.8b, v18.8b, v19.8b},  [x11], #4 // weights_ver
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        zip1            v1.2s,   v1.2s,   v0.2s   // left, flipped
        zip1            v0.2s,   v3.2s,   v2.2s
        zip1            v16.2s,  v16.2s,  v17.2s  // weights_ver
        zip1            v18.2s,  v18.2s,  v19.2s
        shll            v22.8h,  v4.8b,   #8      // bottom*256
        shll            v23.8h,  v4.8b,   #8
        usubl           v0.8h,   v0.8b,   v5.8b   // left-right
        usubl           v1.8h,   v1.8b,   v5.8b
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v18.8h,  v18.8b
        mla             v20.8h,  v0.8h,   v7.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v1.8h,   v7.8h
        mla             v22.8h,  v6.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v23.8h,  v6.8h,   v18.8h
        uhadd           v20.8h,  v20.8h,  v22.8h
        uhadd           v21.8h,  v21.8h,  v23.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn           v21.8b,  v21.8h,  #8
        st1             {v20.s}[0], [x0], x1
        st1             {v20.s}[1], [x6], x1
        subs            w4,  w4,  #4
        st1             {v21.s}[0], [x0], x1
        st1             {v21.s}[1], [x6], x1
        b.gt            4b
        ret
80:
        sub             x2,  x2,  #4
        mov             x7,  #-4
        ld1             {v6.8b}, [x8]             // top
        ld1             {v7.8b}, [x10]            // weights_hor
        dup             v5.16b,  v6.b[7]          // right
        usubl           v6.8h,   v6.8b,   v4.8b   // top-bottom
        uxtl            v7.8h,   v7.8b            // weights_hor
8:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7 // left
        ld4r            {v16.8b, v17.8b, v18.8b, v19.8b},  [x11], #4 // weights_ver
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        shll            v22.8h,  v5.8b,   #8
        shll            v23.8h,  v5.8b,   #8
        usubl           v0.8h,   v0.8b,   v5.8b   // left-right
        usubl           v1.8h,   v1.8b,   v5.8b
        usubl           v2.8h,   v2.8b,   v5.8b
        usubl           v3.8h,   v3.8b,   v5.8b
        shll            v24.8h,  v4.8b,   #8      // bottom*256
        shll            v25.8h,  v4.8b,   #8
        shll            v26.8h,  v4.8b,   #8
        shll            v27.8h,  v4.8b,   #8
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v17.8h,  v17.8b
        uxtl            v18.8h,  v18.8b
        uxtl            v19.8h,  v19.8b
        mla             v20.8h,  v3.8h,   v7.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v2.8h,   v7.8h   // (left flipped)
        mla             v22.8h,  v1.8h,   v7.8h
        mla             v23.8h,  v0.8h,   v7.8h
        mla             v24.8h,  v6.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v25.8h,  v6.8h,   v17.8h
        mla             v26.8h,  v6.8h,   v18.8h
        mla             v27.8h,  v6.8h,   v19.8h
        uhadd           v20.8h,  v20.8h,  v24.8h
        uhadd           v21.8h,  v21.8h,  v25.8h
        uhadd           v22.8h,  v22.8h,  v26.8h
        uhadd           v23.8h,  v23.8h,  v27.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn           v21.8b,  v21.8h,  #8
        rshrn           v22.8b,  v22.8h,  #8
        rshrn           v23.8b,  v23.8h,  #8
        st1             {v20.8b}, [x0], x1
        st1             {v21.8b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v22.8b}, [x0], x1
        st1             {v23.8b}, [x6], x1
        b.gt            8b
        ret
160:
320:
640:
        add             x12, x2,  w3, uxtw
        sub             x2,  x2,  #2
        mov             x7,  #-2
        ld1r            {v5.16b}, [x12]           // right
        sub             x1,  x1,  w3, uxtw
        mov             w9,  w3

1:
        ld2r            {v0.8b, v1.8b},   [x2],  x7 // left
        ld2r            {v16.8b, v17.8b}, [x11], #2 // weights_ver
        usubl           v0.8h,   v0.8b,   v5.8b   // left-right
        usubl           v1.8h,   v1.8b,   v5.8b
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v17.8h,  v17.8b
2:
        ld1             {v7.16b}, [x10],  #16     // weights_hor
        ld1             {v3.16b}, [x8],   #16     // top
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        shll            v22.8h,  v5.8b,   #8
        shll            v23.8h,  v5.8b,   #8
        uxtl            v6.8h,   v7.8b            // weights_hor
        uxtl2           v7.8h,   v7.16b
        usubl           v2.8h,   v3.8b,   v4.8b   // top-bottom
        usubl2          v3.8h,   v3.16b,  v4.16b
        mla             v20.8h,  v1.8h,   v6.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v1.8h,   v7.8h   // (left flipped)
        mla             v22.8h,  v0.8h,   v6.8h
        mla             v23.8h,  v0.8h,   v7.8h
        shll            v24.8h,  v4.8b,   #8      // bottom*256
        shll            v25.8h,  v4.8b,   #8
        shll            v26.8h,  v4.8b,   #8
        shll            v27.8h,  v4.8b,   #8
        mla             v24.8h,  v2.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v25.8h,  v3.8h,   v16.8h
        mla             v26.8h,  v2.8h,   v17.8h
        mla             v27.8h,  v3.8h,   v17.8h
        uhadd           v20.8h,  v20.8h,  v24.8h
        uhadd           v21.8h,  v21.8h,  v25.8h
        uhadd           v22.8h,  v22.8h,  v26.8h
        uhadd           v23.8h,  v23.8h,  v27.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn2          v20.16b, v21.8h,  #8
        rshrn           v22.8b,  v22.8h,  #8
        rshrn2          v22.16b, v23.8h,  #8
        subs            w3,  w3,  #16
        st1             {v20.16b}, [x0],  #16
        st1             {v22.16b}, [x6],  #16
        b.gt            2b
        subs            w4,  w4,  #2
        b.le            9f
        sub             x8,  x8,  w9, uxtw
        sub             x10, x10, w9, uxtw
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        mov             w3,  w9
        b               1b
9:
        ret

L(ipred_smooth_tbl):
        .hword L(ipred_smooth_tbl) - 640b
        .hword L(ipred_smooth_tbl) - 320b
        .hword L(ipred_smooth_tbl) - 160b
        .hword L(ipred_smooth_tbl) -  80b
        .hword L(ipred_smooth_tbl) -  40b
endfunc

// void ipred_smooth_v_neon(pixel *dst, const ptrdiff_t stride,
//                          const pixel *const topleft,
//                          const int width, const int height, const int a,
//                          const int max_width, const int max_height);
function ipred_smooth_v_neon, export=1
        movrel          x7,  X(sm_weights)
        add             x7,  x7,  w4, uxtw
        clz             w9,  w3
        adr             x5,  L(ipred_smooth_v_tbl)
        sub             x8,  x2,  w4, uxtw
        sub             w9,  w9,  #25
        ldrh            w9,  [x5, w9, uxtw #1]
        ld1r            {v4.16b},  [x8] // bottom
        add             x2,  x2,  #1
        sub             x5,  x5,  w9, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        ld1r            {v6.2s}, [x2]             // top
        usubl           v6.8h,   v6.8b,   v4.8b   // top-bottom
4:
        ld4r            {v16.8b, v17.8b, v18.8b, v19.8b},  [x7], #4 // weights_ver
        shll            v22.8h,  v4.8b,   #8      // bottom*256
        shll            v23.8h,  v4.8b,   #8
        zip1            v16.2s,  v16.2s,  v17.2s  // weights_ver
        zip1            v18.2s,  v18.2s,  v19.2s
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v18.8h,  v18.8b
        mla             v22.8h,  v6.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v23.8h,  v6.8h,   v18.8h
        rshrn           v22.8b,  v22.8h,  #8
        rshrn           v23.8b,  v23.8h,  #8
        st1             {v22.s}[0], [x0], x1
        st1             {v22.s}[1], [x6], x1
        subs            w4,  w4,  #4
        st1             {v23.s}[0], [x0], x1
        st1             {v23.s}[1], [x6], x1
        b.gt            4b
        ret
80:
        ld1             {v6.8b}, [x2]             // top
        usubl           v6.8h,   v6.8b,   v4.8b   // top-bottom
8:
        ld4r            {v16.8b, v17.8b, v18.8b, v19.8b},  [x7], #4 // weights_ver
        shll            v24.8h,  v4.8b,   #8      // bottom*256
        shll            v25.8h,  v4.8b,   #8
        shll            v26.8h,  v4.8b,   #8
        shll            v27.8h,  v4.8b,   #8
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v17.8h,  v17.8b
        uxtl            v18.8h,  v18.8b
        uxtl            v19.8h,  v19.8b
        mla             v24.8h,  v6.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v25.8h,  v6.8h,   v17.8h
        mla             v26.8h,  v6.8h,   v18.8h
        mla             v27.8h,  v6.8h,   v19.8h
        rshrn           v24.8b,  v24.8h,  #8
        rshrn           v25.8b,  v25.8h,  #8
        rshrn           v26.8b,  v26.8h,  #8
        rshrn           v27.8b,  v27.8h,  #8
        st1             {v24.8b}, [x0], x1
        st1             {v25.8b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v26.8b}, [x0], x1
        st1             {v27.8b}, [x6], x1
        b.gt            8b
        ret
160:
320:
640:
        // Set up pointers for four rows in parallel; x0, x6, x5, x8
        add             x5,  x0,  x1
        add             x8,  x6,  x1
        lsl             x1,  x1,  #1
        sub             x1,  x1,  w3, uxtw
        mov             w9,  w3

1:
        ld4r            {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver
        uxtl            v16.8h,  v16.8b           // weights_ver
        uxtl            v17.8h,  v17.8b
        uxtl            v18.8h,  v18.8b
        uxtl            v19.8h,  v19.8b
2:
        ld1             {v3.16b}, [x2],   #16     // top
        shll            v20.8h,  v4.8b,   #8      // bottom*256
        shll            v21.8h,  v4.8b,   #8
        shll            v22.8h,  v4.8b,   #8
        shll            v23.8h,  v4.8b,   #8
        shll            v24.8h,  v4.8b,   #8
        shll            v25.8h,  v4.8b,   #8
        shll            v26.8h,  v4.8b,   #8
        shll            v27.8h,  v4.8b,   #8
        usubl           v2.8h,   v3.8b,   v4.8b   // top-bottom
        usubl2          v3.8h,   v3.16b,  v4.16b
        mla             v20.8h,  v2.8h,   v16.8h  // bottom*256 + (top-bottom)*weights_ver
        mla             v21.8h,  v3.8h,   v16.8h
        mla             v22.8h,  v2.8h,   v17.8h
        mla             v23.8h,  v3.8h,   v17.8h
        mla             v24.8h,  v2.8h,   v18.8h
        mla             v25.8h,  v3.8h,   v18.8h
        mla             v26.8h,  v2.8h,   v19.8h
        mla             v27.8h,  v3.8h,   v19.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn2          v20.16b, v21.8h,  #8
        rshrn           v22.8b,  v22.8h,  #8
        rshrn2          v22.16b, v23.8h,  #8
        rshrn           v24.8b,  v24.8h,  #8
        rshrn2          v24.16b, v25.8h,  #8
        rshrn           v26.8b,  v26.8h,  #8
        rshrn2          v26.16b, v27.8h,  #8
        subs            w3,  w3,  #16
        st1             {v20.16b}, [x0],  #16
        st1             {v22.16b}, [x6],  #16
        st1             {v24.16b}, [x5],  #16
        st1             {v26.16b}, [x8],  #16
        b.gt            2b
        subs            w4,  w4,  #4
        b.le            9f
        sub             x2,  x2,  w9, uxtw
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        add             x5,  x5,  x1
        add             x8,  x8,  x1
        mov             w3,  w9
        b               1b
9:
        ret

L(ipred_smooth_v_tbl):
        .hword L(ipred_smooth_v_tbl) - 640b
        .hword L(ipred_smooth_v_tbl) - 320b
        .hword L(ipred_smooth_v_tbl) - 160b
        .hword L(ipred_smooth_v_tbl) -  80b
        .hword L(ipred_smooth_v_tbl) -  40b
endfunc

// void ipred_smooth_h_neon(pixel *dst, const ptrdiff_t stride,
//                          const pixel *const topleft,
//                          const int width, const int height, const int a,
//                          const int max_width, const int max_height);
function ipred_smooth_h_neon, export=1
        movrel          x8,  X(sm_weights)
        add             x8,  x8,  w3, uxtw
        clz             w9,  w3
        adr             x5,  L(ipred_smooth_h_tbl)
        add             x12, x2,  w3, uxtw
        sub             w9,  w9,  #25
        ldrh            w9,  [x5, w9, uxtw #1]
        ld1r            {v5.16b},  [x12] // right
        sub             x5,  x5,  w9, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x5
40:
        ld1r            {v7.2s}, [x8]             // weights_hor
        sub             x2,  x2,  #4
        mov             x7,  #-4
        uxtl            v7.8h,   v7.8b            // weights_hor
4:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7 // left
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        zip1            v1.2s,   v1.2s,   v0.2s   // left, flipped
        zip1            v0.2s,   v3.2s,   v2.2s
        usubl           v0.8h,   v0.8b,   v5.8b   // left-right
        usubl           v1.8h,   v1.8b,   v5.8b
        mla             v20.8h,  v0.8h,   v7.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v1.8h,   v7.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn           v21.8b,  v21.8h,  #8
        st1             {v20.s}[0], [x0], x1
        st1             {v20.s}[1], [x6], x1
        subs            w4,  w4,  #4
        st1             {v21.s}[0], [x0], x1
        st1             {v21.s}[1], [x6], x1
        b.gt            4b
        ret
80:
        ld1             {v7.8b}, [x8]             // weights_hor
        sub             x2,  x2,  #4
        mov             x7,  #-4
        uxtl            v7.8h,   v7.8b            // weights_hor
8:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},  [x2], x7 // left
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        shll            v22.8h,  v5.8b,   #8
        shll            v23.8h,  v5.8b,   #8
        usubl           v3.8h,   v3.8b,   v5.8b   // left-right
        usubl           v2.8h,   v2.8b,   v5.8b
        usubl           v1.8h,   v1.8b,   v5.8b
        usubl           v0.8h,   v0.8b,   v5.8b
        mla             v20.8h,  v3.8h,   v7.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v2.8h,   v7.8h   // (left flipped)
        mla             v22.8h,  v1.8h,   v7.8h
        mla             v23.8h,  v0.8h,   v7.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn           v21.8b,  v21.8h,  #8
        rshrn           v22.8b,  v22.8h,  #8
        rshrn           v23.8b,  v23.8h,  #8
        st1             {v20.8b}, [x0], x1
        st1             {v21.8b}, [x6], x1
        subs            w4,  w4,  #4
        st1             {v22.8b}, [x0], x1
        st1             {v23.8b}, [x6], x1
        b.gt            8b
        ret
160:
320:
640:
        sub             x2,  x2,  #4
        mov             x7,  #-4
        // Set up pointers for four rows in parallel; x0, x6, x5, x10
        add             x5,  x0,  x1
        add             x10, x6,  x1
        lsl             x1,  x1,  #1
        sub             x1,  x1,  w3, uxtw
        mov             w9,  w3

1:
        ld4r            {v0.8b, v1.8b, v2.8b, v3.8b},   [x2],  x7 // left
        usubl           v0.8h,   v0.8b,   v5.8b   // left-right
        usubl           v1.8h,   v1.8b,   v5.8b
        usubl           v2.8h,   v2.8b,   v5.8b
        usubl           v3.8h,   v3.8b,   v5.8b
2:
        ld1             {v7.16b}, [x8],   #16     // weights_hor
        shll            v20.8h,  v5.8b,   #8      // right*256
        shll            v21.8h,  v5.8b,   #8
        shll            v22.8h,  v5.8b,   #8
        shll            v23.8h,  v5.8b,   #8
        shll            v24.8h,  v5.8b,   #8
        shll            v25.8h,  v5.8b,   #8
        shll            v26.8h,  v5.8b,   #8
        shll            v27.8h,  v5.8b,   #8
        uxtl            v6.8h,   v7.8b            // weights_hor
        uxtl2           v7.8h,   v7.16b
        mla             v20.8h,  v3.8h,   v6.8h   // right*256  + (left-right)*weights_hor
        mla             v21.8h,  v3.8h,   v7.8h   // (left flipped)
        mla             v22.8h,  v2.8h,   v6.8h
        mla             v23.8h,  v2.8h,   v7.8h
        mla             v24.8h,  v1.8h,   v6.8h
        mla             v25.8h,  v1.8h,   v7.8h
        mla             v26.8h,  v0.8h,   v6.8h
        mla             v27.8h,  v0.8h,   v7.8h
        rshrn           v20.8b,  v20.8h,  #8
        rshrn2          v20.16b, v21.8h,  #8
        rshrn           v22.8b,  v22.8h,  #8
        rshrn2          v22.16b, v23.8h,  #8
        rshrn           v24.8b,  v24.8h,  #8
        rshrn2          v24.16b, v25.8h,  #8
        rshrn           v26.8b,  v26.8h,  #8
        rshrn2          v26.16b, v27.8h,  #8
        subs            w3,  w3,  #16
        st1             {v20.16b}, [x0],  #16
        st1             {v22.16b}, [x6],  #16
        st1             {v24.16b}, [x5],  #16
        st1             {v26.16b}, [x10], #16
        b.gt            2b
        subs            w4,  w4,  #4
        b.le            9f
        sub             x8,  x8,  w9, uxtw
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        add             x5,  x5,  x1
        add             x10, x10, x1
        mov             w3,  w9
        b               1b
9:
        ret

L(ipred_smooth_h_tbl):
        .hword L(ipred_smooth_h_tbl) - 640b
        .hword L(ipred_smooth_h_tbl) - 320b
        .hword L(ipred_smooth_h_tbl) - 160b
        .hword L(ipred_smooth_h_tbl) -  80b
        .hword L(ipred_smooth_h_tbl) -  40b
endfunc

// void ipred_filter_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int filt_idx,
//                        const int max_width, const int max_height);
function ipred_filter_neon, export=1
        and             w5,  w5,  #511
        movrel          x6,  X(filter_intra_taps)
        lsl             w5,  w5,  #6
        add             x6,  x6,  w5, uxtw
        ld1             {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32
        clz             w9,  w3
        adr             x5,  L(ipred_filter_tbl)
        ld1             {v20.8b, v21.8b, v22.8b}, [x6]
        sub             w9,  w9,  #26
        ldrh            w9,  [x5, w9, uxtw #1]
        sxtl            v16.8h,  v16.8b
        sxtl            v17.8h,  v17.8b
        sub             x5,  x5,  w9, uxtw
        sxtl            v18.8h,  v18.8b
        sxtl            v19.8h,  v19.8b
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        sxtl            v20.8h,  v20.8b
        sxtl            v21.8h,  v21.8b
        sxtl            v22.8h,  v22.8b
        br              x5
40:
        ldur            s0,  [x2, #1]             // top (0-3)
        sub             x2,  x2,  #2
        mov             x7,  #-2
        uxtl            v0.8h,   v0.8b            // top (0-3)
4:
        ld1             {v1.s}[0], [x2], x7       // left (0-1) + topleft (2)
        mul             v2.8h,   v17.8h,  v0.h[0] // p1(top[0]) * filter(1)
        mla             v2.8h,   v18.8h,  v0.h[1] // p2(top[1]) * filter(2)
        mla             v2.8h,   v19.8h,  v0.h[2] // p3(top[2]) * filter(3)
        uxtl            v1.8h,   v1.8b            // left (0-1) + topleft (2)
        mla             v2.8h,   v20.8h,  v0.h[3] // p4(top[3]) * filter(4)
        mla             v2.8h,   v16.8h,  v1.h[2] // p0(topleft) * filter(0)
        mla             v2.8h,   v21.8h,  v1.h[1] // p5(left[0]) * filter(5)
        mla             v2.8h,   v22.8h,  v1.h[0] // p6(left[1]) * filter(6)
        sqrshrun        v2.8b,   v2.8h,   #4
        subs            w4,  w4,  #2
        st1             {v2.s}[0], [x0], x1
        uxtl            v0.8h,   v2.8b
        st1             {v2.s}[1], [x6], x1
        ext             v0.16b,  v0.16b,  v0.16b, #8 // move top from [4-7] to [0-3]
        b.gt            4b
        ret
80:
        ldur            d0,  [x2, #1]             // top (0-7)
        sub             x2,  x2,  #2
        mov             x7,  #-2
        uxtl            v0.8h,   v0.8b            // top (0-7)
8:
        ld1             {v1.s}[0], [x2], x7       // left (0-1) + topleft (2)
        mul             v2.8h,   v17.8h,  v0.h[0] // p1(top[0]) * filter(1)
        mla             v2.8h,   v18.8h,  v0.h[1] // p2(top[1]) * filter(2)
        mla             v2.8h,   v19.8h,  v0.h[2] // p3(top[2]) * filter(3)
        uxtl            v1.8h,   v1.8b            // left (0-1) + topleft (2)
        mla             v2.8h,   v20.8h,  v0.h[3] // p4(top[3]) * filter(4)
        mla             v2.8h,   v16.8h,  v1.h[2] // p0(topleft) * filter(0)
        mla             v2.8h,   v21.8h,  v1.h[1] // p5(left[0]) * filter(5)
        mla             v2.8h,   v22.8h,  v1.h[0] // p6(left[1]) * filter(6)
        mul             v3.8h,   v17.8h,  v0.h[4] // p1(top[0]) * filter(1)
        mla             v3.8h,   v18.8h,  v0.h[5] // p2(top[1]) * filter(2)
        mla             v3.8h,   v19.8h,  v0.h[6] // p3(top[2]) * filter(3)
        sqrshrun        v2.8b,   v2.8h,   #4
        uxtl            v1.8h,   v2.8b            // first block, in 16 bit
        mla             v3.8h,   v20.8h,  v0.h[7] // p4(top[3]) * filter(4)
        mla             v3.8h,   v16.8h,  v0.h[3] // p0(topleft) * filter(0)
        mla             v3.8h,   v21.8h,  v1.h[3] // p5(left[0]) * filter(5)
        mla             v3.8h,   v22.8h,  v1.h[7] // p6(left[1]) * filter(6)
        sqrshrun        v3.8b,   v3.8h,   #4
        subs            w4,  w4,  #2
        st2             {v2.s, v3.s}[0], [x0], x1
        zip2            v0.2s,   v2.2s,   v3.2s
        st2             {v2.s, v3.s}[1], [x6], x1
        uxtl            v0.8h,   v0.8b
        b.gt            8b
        ret
160:
320:
        add             x8,  x2,  #1
        sub             x2,  x2,  #2
        mov             x7,  #-2
        sub             x1,  x1,  w3, uxtw
        mov             w9,  w3

1:
        ld1             {v0.s}[0], [x2], x7       // left (0-1) + topleft (2)
        uxtl            v0.8h,   v0.8b            // left (0-1) + topleft (2)
2:
        ld1             {v2.16b}, [x8],   #16     // top(0-15)
        mul             v3.8h,   v16.8h,  v0.h[2] // p0(topleft) * filter(0)
        mla             v3.8h,   v21.8h,  v0.h[1] // p5(left[0]) * filter(5)
        uxtl            v1.8h,   v2.8b            // top(0-7)
        uxtl2           v2.8h,   v2.16b           // top(8-15)
        mla             v3.8h,   v22.8h,  v0.h[0] // p6(left[1]) * filter(6)
        mla             v3.8h,   v17.8h,  v1.h[0] // p1(top[0]) * filter(1)
        mla             v3.8h,   v18.8h,  v1.h[1] // p2(top[1]) * filter(2)
        mla             v3.8h,   v19.8h,  v1.h[2] // p3(top[2]) * filter(3)
        mla             v3.8h,   v20.8h,  v1.h[3] // p4(top[3]) * filter(4)

        mul             v4.8h,   v17.8h,  v1.h[4] // p1(top[0]) * filter(1)
        mla             v4.8h,   v18.8h,  v1.h[5] // p2(top[1]) * filter(2)
        mla             v4.8h,   v19.8h,  v1.h[6] // p3(top[2]) * filter(3)
        sqrshrun        v3.8b,   v3.8h,   #4
        uxtl            v0.8h,   v3.8b            // first block, in 16 bit
        mla             v4.8h,   v20.8h,  v1.h[7] // p4(top[3]) * filter(4)
        mla             v4.8h,   v16.8h,  v1.h[3] // p0(topleft) * filter(0)
        mla             v4.8h,   v21.8h,  v0.h[3] // p5(left[0]) * filter(5)
        mla             v4.8h,   v22.8h,  v0.h[7] // p6(left[1]) * filter(6)

        mul             v5.8h,   v17.8h,  v2.h[0] // p1(top[0]) * filter(1)
        mla             v5.8h,   v18.8h,  v2.h[1] // p2(top[1]) * filter(2)
        mla             v5.8h,   v19.8h,  v2.h[2] // p3(top[2]) * filter(3)
        sqrshrun        v4.8b,   v4.8h,   #4
        uxtl            v0.8h,   v4.8b            // second block, in 16 bit
        mla             v5.8h,   v20.8h,  v2.h[3] // p4(top[3]) * filter(4)
        mla             v5.8h,   v16.8h,  v1.h[7] // p0(topleft) * filter(0)
        mla             v5.8h,   v21.8h,  v0.h[3] // p5(left[0]) * filter(5)
        mla             v5.8h,   v22.8h,  v0.h[7] // p6(left[1]) * filter(6)

        mul             v6.8h,   v17.8h,  v2.h[4] // p1(top[0]) * filter(1)
        mla             v6.8h,   v18.8h,  v2.h[5] // p2(top[1]) * filter(2)
        mla             v6.8h,   v19.8h,  v2.h[6] // p3(top[2]) * filter(3)
        sqrshrun        v5.8b,   v5.8h,   #4
        uxtl            v0.8h,   v5.8b            // third block, in 16 bit
        mla             v6.8h,   v20.8h,  v2.h[7] // p4(top[3]) * filter(4)
        mla             v6.8h,   v16.8h,  v2.h[3] // p0(topleft) * filter(0)
        mla             v6.8h,   v21.8h,  v0.h[3] // p5(left[0]) * filter(5)
        mla             v6.8h,   v22.8h,  v0.h[7] // p6(left[1]) * filter(6)

        subs            w3,  w3,  #16
        sqrshrun        v6.8b,   v6.8h,   #4

        ins             v0.h[2], v2.h[7]
        st4             {v3.s, v4.s, v5.s, v6.s}[0], [x0], #16
        ins             v0.b[0], v6.b[7]
        st4             {v3.s, v4.s, v5.s, v6.s}[1], [x6], #16
        ins             v0.b[2], v6.b[3]
        b.gt            2b
        subs            w4,  w4,  #2
        b.le            9f
        sub             x8,  x6,  w9, uxtw
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        mov             w3,  w9
        b               1b
9:
        ret

L(ipred_filter_tbl):
        .hword L(ipred_filter_tbl) - 320b
        .hword L(ipred_filter_tbl) - 160b
        .hword L(ipred_filter_tbl) -  80b
        .hword L(ipred_filter_tbl) -  40b
endfunc

// void pal_pred_neon(pixel *dst, const ptrdiff_t stride,
//                    const uint16_t *const pal, const uint8_t *idx,
//                    const int w, const int h);
function pal_pred_neon, export=1
        ld1             {v0.8h}, [x2]
        clz             w9,  w4
        adr             x6,  L(pal_pred_tbl)
        sub             w9,  w9,  #25
        ldrh            w9,  [x6, w9, uxtw #1]
        xtn             v0.8b,  v0.8h
        sub             x6,  x6,  w9, uxtw
        add             x2,  x0,  x1
        lsl             x1,  x1,  #1
        br              x6
4:
        ld1             {v1.16b}, [x3], #16
        subs            w5,  w5,  #4
        tbl             v1.16b, {v0.16b}, v1.16b
        st1             {v1.s}[0], [x0], x1
        st1             {v1.s}[1], [x2], x1
        st1             {v1.s}[2], [x0], x1
        st1             {v1.s}[3], [x2], x1
        b.gt            4b
        ret
8:
        ld1             {v1.16b, v2.16b}, [x3], #32
        subs            w5,  w5,  #4
        tbl             v1.16b, {v0.16b}, v1.16b
        st1             {v1.d}[0], [x0], x1
        tbl             v2.16b, {v0.16b}, v2.16b
        st1             {v1.d}[1], [x2], x1
        st1             {v2.d}[0], [x0], x1
        st1             {v2.d}[1], [x2], x1
        b.gt            8b
        ret
16:
        ld1             {v1.16b, v2.16b, v3.16b, v4.16b}, [x3], #64
        subs            w5,  w5,  #4
        tbl             v1.16b, {v0.16b}, v1.16b
        tbl             v2.16b, {v0.16b}, v2.16b
        st1             {v1.16b}, [x0], x1
        tbl             v3.16b, {v0.16b}, v3.16b
        st1             {v2.16b}, [x2], x1
        tbl             v4.16b, {v0.16b}, v4.16b
        st1             {v3.16b}, [x0], x1
        st1             {v4.16b}, [x2], x1
        b.gt            16b
        ret
32:
        ld1             {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64
        ld1             {v20.16b, v21.16b, v22.16b, v23.16b}, [x3], #64
        subs            w5,  w5,  #4
        tbl             v16.16b, {v0.16b}, v16.16b
        tbl             v17.16b, {v0.16b}, v17.16b
        tbl             v18.16b, {v0.16b}, v18.16b
        tbl             v19.16b, {v0.16b}, v19.16b
        tbl             v20.16b, {v0.16b}, v20.16b
        st1             {v16.16b, v17.16b}, [x0], x1
        tbl             v21.16b, {v0.16b}, v21.16b
        st1             {v18.16b, v19.16b}, [x2], x1
        tbl             v22.16b, {v0.16b}, v22.16b
        st1             {v20.16b, v21.16b}, [x0], x1
        tbl             v23.16b, {v0.16b}, v23.16b
        st1             {v22.16b, v23.16b}, [x2], x1
        b.gt            32b
        ret
64:
        ld1             {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64
        ld1             {v20.16b, v21.16b, v22.16b, v23.16b}, [x3], #64
        subs            w5,  w5,  #2
        tbl             v16.16b, {v0.16b}, v16.16b
        tbl             v17.16b, {v0.16b}, v17.16b
        tbl             v18.16b, {v0.16b}, v18.16b
        tbl             v19.16b, {v0.16b}, v19.16b
        st1             {v16.16b, v17.16b, v18.16b, v19.16b}, [x0], x1
        tbl             v20.16b, {v0.16b}, v20.16b
        tbl             v21.16b, {v0.16b}, v21.16b
        tbl             v22.16b, {v0.16b}, v22.16b
        tbl             v23.16b, {v0.16b}, v23.16b
        st1             {v20.16b, v21.16b, v22.16b, v23.16b}, [x2], x1
        b.gt            64b
        ret

L(pal_pred_tbl):
        .hword L(pal_pred_tbl) - 64b
        .hword L(pal_pred_tbl) - 32b
        .hword L(pal_pred_tbl) - 16b
        .hword L(pal_pred_tbl) -  8b
        .hword L(pal_pred_tbl) -  4b
endfunc

// void ipred_cfl_128_neon(pixel *dst, const ptrdiff_t stride,
//                         const pixel *const topleft,
//                         const int width, const int height,
//                         const int16_t *ac, const int alpha);
function ipred_cfl_128_neon, export=1
        clz             w9,  w3
        adr             x7,  L(ipred_cfl_128_tbl)
        sub             w9,  w9,  #26
        ldrh            w9,  [x7, w9, uxtw #1]
        movi            v0.8h,   #128 // dc
        dup             v1.8h,   w6   // alpha
        sub             x7,  x7,  w9, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x7
L(ipred_cfl_splat_w4):
        ld1             {v2.8h, v3.8h}, [x5], #32
        mul             v2.8h,   v2.8h,   v1.8h  // diff = ac * alpha
        mul             v3.8h,   v3.8h,   v1.8h
        sshr            v4.8h,   v2.8h,   #15    // sign = diff >> 15
        sshr            v5.8h,   v3.8h,   #15
        add             v2.8h,   v2.8h,   v4.8h  // diff + sign
        add             v3.8h,   v3.8h,   v5.8h
        srshr           v2.8h,   v2.8h,   #6     // (diff + sign + 32) >> 6 = apply_sign()
        srshr           v3.8h,   v3.8h,   #6
        add             v2.8h,   v2.8h,   v0.8h  // dc + apply_sign()
        add             v3.8h,   v3.8h,   v0.8h
        sqxtun          v2.8b,   v2.8h           // iclip_pixel(dc + apply_sign())
        sqxtun          v3.8b,   v3.8h
        st1             {v2.s}[0],  [x0], x1
        st1             {v2.s}[1],  [x6], x1
        subs            w4,  w4,  #4
        st1             {v3.s}[0],  [x0], x1
        st1             {v3.s}[1],  [x6], x1
        b.gt            L(ipred_cfl_splat_w4)
        ret
L(ipred_cfl_splat_w8):
        ld1             {v2.8h, v3.8h, v4.8h, v5.8h}, [x5], #64
        mul             v2.8h,   v2.8h,   v1.8h  // diff = ac * alpha
        mul             v3.8h,   v3.8h,   v1.8h
        mul             v4.8h,   v4.8h,   v1.8h
        mul             v5.8h,   v5.8h,   v1.8h
        sshr            v16.8h,  v2.8h,   #15    // sign = diff >> 15
        sshr            v17.8h,  v3.8h,   #15
        sshr            v18.8h,  v4.8h,   #15
        sshr            v19.8h,  v5.8h,   #15
        add             v2.8h,   v2.8h,   v16.8h // diff + sign
        add             v3.8h,   v3.8h,   v17.8h
        add             v4.8h,   v4.8h,   v18.8h
        add             v5.8h,   v5.8h,   v19.8h
        srshr           v2.8h,   v2.8h,   #6     // (diff + sign + 32) >> 6 = apply_sign()
        srshr           v3.8h,   v3.8h,   #6
        srshr           v4.8h,   v4.8h,   #6
        srshr           v5.8h,   v5.8h,   #6
        add             v2.8h,   v2.8h,   v0.8h  // dc + apply_sign()
        add             v3.8h,   v3.8h,   v0.8h
        add             v4.8h,   v4.8h,   v0.8h
        add             v5.8h,   v5.8h,   v0.8h
        sqxtun          v2.8b,   v2.8h           // iclip_pixel(dc + apply_sign())
        sqxtun          v3.8b,   v3.8h
        sqxtun          v4.8b,   v4.8h
        sqxtun          v5.8b,   v5.8h
        st1             {v2.8b},  [x0], x1
        st1             {v3.8b},  [x6], x1
        subs            w4,  w4,  #4
        st1             {v4.8b},  [x0], x1
        st1             {v5.8b},  [x6], x1
        b.gt            L(ipred_cfl_splat_w8)
        ret
L(ipred_cfl_splat_w16):
        add             x7,  x5,  w3, uxtw #1
        sub             x1,  x1,  w3, uxtw
        mov             w9,  w3
1:
        ld1             {v2.8h, v3.8h}, [x5], #32
        ld1             {v4.8h, v5.8h}, [x7], #32
        mul             v2.8h,   v2.8h,   v1.8h  // diff = ac * alpha
        mul             v3.8h,   v3.8h,   v1.8h
        mul             v4.8h,   v4.8h,   v1.8h
        mul             v5.8h,   v5.8h,   v1.8h
        sshr            v16.8h,  v2.8h,   #15    // sign = diff >> 15
        sshr            v17.8h,  v3.8h,   #15
        sshr            v18.8h,  v4.8h,   #15
        sshr            v19.8h,  v5.8h,   #15
        add             v2.8h,   v2.8h,   v16.8h // diff + sign
        add             v3.8h,   v3.8h,   v17.8h
        add             v4.8h,   v4.8h,   v18.8h
        add             v5.8h,   v5.8h,   v19.8h
        srshr           v2.8h,   v2.8h,   #6     // (diff + sign + 32) >> 6 = apply_sign()
        srshr           v3.8h,   v3.8h,   #6
        srshr           v4.8h,   v4.8h,   #6
        srshr           v5.8h,   v5.8h,   #6
        add             v2.8h,   v2.8h,   v0.8h  // dc + apply_sign()
        add             v3.8h,   v3.8h,   v0.8h
        add             v4.8h,   v4.8h,   v0.8h
        add             v5.8h,   v5.8h,   v0.8h
        sqxtun          v2.8b,   v2.8h           // iclip_pixel(dc + apply_sign())
        sqxtun          v3.8b,   v3.8h
        sqxtun          v4.8b,   v4.8h
        sqxtun          v5.8b,   v5.8h
        subs            w3,  w3,  #16
        st1             {v2.8b, v3.8b},  [x0], #16
        st1             {v4.8b, v5.8b},  [x6], #16
        b.gt            1b
        subs            w4,  w4,  #2
        add             x5,  x5,  w9, uxtw #1
        add             x7,  x7,  w9, uxtw #1
        add             x0,  x0,  x1
        add             x6,  x6,  x1
        mov             w3,  w9
        b.gt            1b
        ret

L(ipred_cfl_128_tbl):
L(ipred_cfl_splat_tbl):
        .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16)
        .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16)
        .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8)
        .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4)
endfunc

// void ipred_cfl_top_neon(pixel *dst, const ptrdiff_t stride,
//                         const pixel *const topleft,
//                         const int width, const int height,
//                         const int16_t *ac, const int alpha);
function ipred_cfl_top_neon, export=1
        clz             w9,  w3
        adr             x7,  L(ipred_cfl_top_tbl)
        sub             w9,  w9,  #26
        ldrh            w9,  [x7, w9, uxtw #1]
        dup             v1.8h,   w6   // alpha
        add             x2,  x2,  #1
        sub             x7,  x7,  w9, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x7
4:
        ld1r            {v0.2s},  [x2]
        uaddlv          h0,      v0.8b
        urshr           v0.8h,   v0.8h,   #3
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w4)
8:
        ld1             {v0.8b},  [x2]
        uaddlv          h0,      v0.8b
        urshr           v0.8h,   v0.8h,   #3
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w8)
16:
        ld1             {v0.16b}, [x2]
        uaddlv          h0,      v0.16b
        urshr           v0.8h,   v0.8h,   #4
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w16)
32:
        ld1             {v2.16b, v3.16b}, [x2]
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v2.4h,   v2.4h,   v3.4h
        urshr           v2.8h,   v2.8h,   #5
        dup             v0.8h,   v2.h[0]
        b               L(ipred_cfl_splat_w16)

L(ipred_cfl_top_tbl):
        .hword L(ipred_cfl_top_tbl) - 32b
        .hword L(ipred_cfl_top_tbl) - 16b
        .hword L(ipred_cfl_top_tbl) -  8b
        .hword L(ipred_cfl_top_tbl) -  4b
endfunc

// void ipred_cfl_left_neon(pixel *dst, const ptrdiff_t stride,
//                          const pixel *const topleft,
//                          const int width, const int height,
//                          const int16_t *ac, const int alpha);
function ipred_cfl_left_neon, export=1
        sub             x2,  x2,  w4, uxtw
        clz             w9,  w3
        clz             w8,  w4
        adr             x10, L(ipred_cfl_splat_tbl)
        adr             x7,  L(ipred_cfl_left_tbl)
        sub             w9,  w9,  #26
        sub             w8,  w8,  #26
        ldrh            w9,  [x10, w9, uxtw #1]
        ldrh            w8,  [x7,  w8, uxtw #1]
        dup             v1.8h,   w6   // alpha
        sub             x9,  x10, w9, uxtw
        sub             x7,  x7,  w8, uxtw
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x7

L(ipred_cfl_left_h4):
        ld1r            {v0.2s},  [x2]
        uaddlv          h0,      v0.8b
        urshr           v0.8h,   v0.8h,   #3
        dup             v0.8h,   v0.h[0]
        br              x9

L(ipred_cfl_left_h8):
        ld1             {v0.8b},  [x2]
        uaddlv          h0,      v0.8b
        urshr           v0.8h,   v0.8h,   #3
        dup             v0.8h,   v0.h[0]
        br              x9

L(ipred_cfl_left_h16):
        ld1             {v0.16b}, [x2]
        uaddlv          h0,      v0.16b
        urshr           v0.8h,   v0.8h,   #4
        dup             v0.8h,   v0.h[0]
        br              x9

L(ipred_cfl_left_h32):
        ld1             {v2.16b, v3.16b}, [x2]
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v2.4h,   v2.4h,   v3.4h
        urshr           v2.8h,   v2.8h,   #5
        dup             v0.8h,   v2.h[0]
        br              x9

L(ipred_cfl_left_tbl):
        .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32)
        .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16)
        .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8)
        .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4)
endfunc

// void ipred_cfl_neon(pixel *dst, const ptrdiff_t stride,
//                     const pixel *const topleft,
//                     const int width, const int height,
//                     const int16_t *ac, const int alpha);
function ipred_cfl_neon, export=1
        sub             x2,  x2,  w4, uxtw
        add             w8,  w3,  w4             // width + height
        dup             v1.8h,   w6              // alpha
        clz             w9,  w3
        clz             w6,  w4
        dup             v16.8h, w8               // width + height
        adr             x7,  L(ipred_cfl_tbl)
        rbit            w8,  w8                  // rbit(width + height)
        sub             w9,  w9,  #22            // 22 leading bits, minus table offset 4
        sub             w6,  w6,  #26
        clz             w8,  w8                  // ctz(width + height)
        ldrh            w9,  [x7, w9, uxtw #1]
        ldrh            w6,  [x7, w6, uxtw #1]
        neg             w8,  w8                  // -ctz(width + height)
        sub             x9,  x7,  w9, uxtw
        sub             x7,  x7,  w6, uxtw
        ushr            v16.8h,  v16.8h,  #1     // (width + height) >> 1
        dup             v17.8h,  w8              // -ctz(width + height)
        add             x6,  x0,  x1
        lsl             x1,  x1,  #1
        br              x7

L(ipred_cfl_h4):
        ld1             {v0.s}[0],  [x2], #4
        ins             v0.s[1], wzr
        uaddlv          h0,      v0.8b
        br              x9
L(ipred_cfl_w4):
        add             x2,  x2,  #1
        ld1             {v2.s}[0],  [x2]
        ins             v2.s[1], wzr
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h2,      v2.8b
        cmp             w4,  #4
        add             v0.4h,   v0.4h,   v2.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 8/16
        mov             w16, #(0x3334/2)
        movk            w16, #(0x5556/2), lsl #16
        add             w17, w4,  w4  // w17 = 2*h = 16 or 32
        lsr             w16, w16, w17
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w4)

L(ipred_cfl_h8):
        ld1             {v0.8b},  [x2], #8
        uaddlv          h0,      v0.8b
        br              x9
L(ipred_cfl_w8):
        add             x2,  x2,  #1
        ld1             {v2.8b},  [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h2,      v2.8b
        cmp             w4,  #8
        add             v0.4h,   v0.4h,   v2.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 4/16/32
        cmp             w4,  #32
        mov             w16, #(0x3334/2)
        mov             w17, #(0x5556/2)
        csel            w16, w16, w17, eq
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w8)

L(ipred_cfl_h16):
        ld1             {v0.16b}, [x2], #16
        uaddlv          h0,      v0.16b
        br              x9
L(ipred_cfl_w16):
        add             x2,  x2,  #1
        ld1             {v2.16b}, [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h2,      v2.16b
        cmp             w4,  #16
        add             v0.4h,   v0.4h,   v2.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 4/8/32
        cmp             w4,  #4
        mov             w16, #(0x3334/2)
        mov             w17, #(0x5556/2)
        csel            w16, w16, w17, eq
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w16)

L(ipred_cfl_h32):
        ld1             {v2.16b, v3.16b}, [x2], #32
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        add             v0.4h,   v2.4h,   v3.4h
        br              x9
L(ipred_cfl_w32):
        add             x2,  x2,  #1
        ld1             {v2.16b, v3.16b}, [x2]
        add             v0.4h,   v0.4h,   v16.4h
        uaddlv          h2,      v2.16b
        uaddlv          h3,      v3.16b
        cmp             w4,  #32
        add             v0.4h,   v0.4h,   v2.4h
        add             v0.4h,   v0.4h,   v3.4h
        ushl            v0.4h,   v0.4h,   v17.4h
        b.eq            1f
        // h = 8/16
        mov             w16, #(0x5556/2)
        movk            w16, #(0x3334/2), lsl #16
        add             w17, w4,  w4  // w17 = 2*h = 16 or 32
        lsr             w16, w16, w17
        dup             v16.4h,  w16
        sqdmulh         v0.4h,   v0.4h,   v16.4h
1:
        dup             v0.8h,   v0.h[0]
        b               L(ipred_cfl_splat_w16)

L(ipred_cfl_tbl):
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_h32)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_h16)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_h8)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_h4)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_w32)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8)
        .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4)
endfunc

// void cfl_ac_420_neon(int16_t *const ac, const pixel *const ypx,
//                      const ptrdiff_t stride, const int w_pad,
//                      const int h_pad, const int cw, const int ch);
function ipred_cfl_ac_420_neon, export=1
        clz             w8,  w5
        lsl             w4,  w4,  #2
        adr             x7,  L(ipred_cfl_ac_420_tbl)
        sub             w8,  w8,  #27
        ldrh            w8,  [x7, w8, uxtw #1]
        sub             x7,  x7,  w8, uxtw
        sub             w8,  w6,  w4         // height - h_pad
        rbit            w9,  w5              // rbit(width)
        rbit            w10, w6              // rbit(height)
        clz             w9,  w9              // ctz(width)
        clz             w10, w10             // ctz(height)
        add             w9,  w9,  w10        // log2sz
        movi            v16.4s,  #1
        add             x10, x1,  x2
        lsl             x2,  x2,  #1
        dup             v17.4s,  w9
        sshl            v16.4s,  v16.4s,  v17.4s // 1 << log2sz
        neg             v17.4s,  v17.4s          // -log2sz
        ushr            v16.4s,  v16.4s,  #1     // 1 << (log2sz - 1)
        mov             w9,  w6
        br              x7

L(ipred_cfl_ac_420_w4):
1:      // Copy and subsample input
        ld1             {v0.8b},   [x1],  x2
        ld1             {v1.8b},   [x10], x2
        ld1             {v0.d}[1], [x1],  x2
        ld1             {v1.d}[1], [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v1.8h,   v1.16b
        add             v0.8h,   v0.8h,   v1.8h
        shl             v0.8h,   v0.8h,   #1
        subs            w8,  w8,  #2
        st1             {v0.8h}, [x0], #16
        b.gt            1b
        trn2            v1.2d,   v0.2d,   v0.2d
        trn2            v0.2d,   v0.2d,   v0.2d
L(ipred_cfl_ac_420_w4_hpad):
        cbz             w4,  3f
2:      // Vertical padding (h_pad > 0)
        subs            w4,  w4,  #4
        st1             {v0.8h, v1.8h}, [x0], #32
        b.gt            2b
3:
        sub             x0,  x0,  w6, uxtw #3
        // Sum the produced ac values
        subs            w6,  w6,  #4
        ld1             {v0.8h, v1.8h}, [x0], #32
        b.le            5f
4:
        ld1             {v2.8h, v3.8h}, [x0], #32
        subs            w6,  w6,  #4
        add             v0.8h,   v0.8h,   v2.8h
        add             v1.8h,   v1.8h,   v3.8h
        b.gt            4b
5:
        add             v0.8h,   v0.8h,   v1.8h
        uaddlv          s0,  v0.8h                // sum
        sub             x0,  x0,  w9, uxtw #3
        add             v0.2s,   v0.2s,   v16.2s  // sum += 1 << (log2sz - 1)
        ushl            v4.2s,   v0.2s,   v17.2s  // sum >>= log2sz
        dup             v4.8h,   v4.h[0]
6:      // Subtract dc from ac
        ld1             {v0.8h, v1.8h}, [x0]
        subs            w9,  w9,  #4
        sub             v0.8h,   v0.8h,   v4.8h
        sub             v1.8h,   v1.8h,   v4.8h
        st1             {v0.8h, v1.8h}, [x0], #32
        b.gt            6b
        ret

L(ipred_cfl_ac_420_w8):
        cbnz            w3,  L(ipred_cfl_ac_420_w8_wpad)
1:      // Copy and subsample input, without padding
        ld1             {v0.16b}, [x1],  x2
        ld1             {v1.16b}, [x10], x2
        ld1             {v2.16b}, [x1],  x2
        uaddlp          v0.8h,   v0.16b
        ld1             {v3.16b}, [x10], x2
        uaddlp          v1.8h,   v1.16b
        uaddlp          v2.8h,   v2.16b
        uaddlp          v3.8h,   v3.16b
        add             v0.8h,   v0.8h,   v1.8h
        add             v2.8h,   v2.8h,   v3.8h
        shl             v0.8h,   v0.8h,   #1
        shl             v1.8h,   v2.8h,   #1
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h}, [x0], #32
        b.gt            1b
        mov             v0.16b,  v1.16b
        b               L(ipred_cfl_ac_420_w8_hpad)

L(ipred_cfl_ac_420_w8_wpad):
1:      // Copy and subsample input, padding 4
        ld1             {v0.8b},   [x1],  x2
        ld1             {v1.8b},   [x10], x2
        ld1             {v0.d}[1], [x1],  x2
        ld1             {v1.d}[1], [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v1.8h,   v1.16b
        add             v0.8h,   v0.8h,   v1.8h
        shl             v0.8h,   v0.8h,   #1
        dup             v1.4h,   v0.h[3]
        dup             v3.4h,   v0.h[7]
        trn2            v2.2d,   v0.2d,   v0.2d
        subs            w8,  w8,  #2
        st1             {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32
        b.gt            1b
        trn1            v0.2d,   v2.2d,   v3.2d
        trn1            v1.2d,   v2.2d,   v3.2d

L(ipred_cfl_ac_420_w8_hpad):
        cbz             w4,  3f
2:      // Vertical padding (h_pad > 0)
        subs            w4,  w4,  #4
        st1             {v0.8h, v1.8h}, [x0], #32
        st1             {v0.8h, v1.8h}, [x0], #32
        b.gt            2b
3:

L(ipred_cfl_ac_420_w8_calc_subtract_dc):
        sub             x0,  x0,  w6, uxtw #4
        // Sum the produced ac values
        subs            w6,  w6,  #4
        ld1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.le            5f
4:
        ld1             {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64
        subs            w6,  w6,  #4
        add             v0.8h,   v0.8h,   v4.8h
        add             v1.8h,   v1.8h,   v5.8h
        add             v2.8h,   v2.8h,   v6.8h
        add             v3.8h,   v3.8h,   v7.8h
        b.gt            4b
5:
        add             v0.8h,   v0.8h,   v1.8h
        add             v2.8h,   v2.8h,   v3.8h
        uaddlp          v0.4s,   v0.8h
        uaddlp          v2.4s,   v2.8h
        add             v0.4s,   v0.4s,   v2.4s
        addv            s0,  v0.4s                // sum
        sub             x0,  x0,  w9, uxtw #4
        add             v0.2s,   v0.2s,   v16.2s  // sum += 1 << (log2sz - 1)
        ushl            v4.2s,   v0.2s,   v17.2s  // sum >>= log2sz
        dup             v4.8h,   v4.h[0]
6:      // Subtract dc from ac
        ld1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0]
        subs            w9,  w9,  #4
        sub             v0.8h,   v0.8h,   v4.8h
        sub             v1.8h,   v1.8h,   v4.8h
        sub             v2.8h,   v2.8h,   v4.8h
        sub             v3.8h,   v3.8h,   v4.8h
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            6b
        ret

L(ipred_cfl_ac_420_w16):
        adr             x7,  L(ipred_cfl_ac_420_w16_tbl)
        ldrh            w3,  [x7, w3, uxtw #1]
        sub             x7,  x7,  w3, uxtw
        br              x7

L(ipred_cfl_ac_420_w16_wpad0):
1:      // Copy and subsample input, without padding
        ld1             {v0.16b, v1.16b}, [x1],  x2
        ld1             {v2.16b, v3.16b}, [x10], x2
        uaddlp          v0.8h,   v0.16b
        ld1             {v4.16b, v5.16b}, [x1],  x2
        uaddlp          v1.8h,   v1.16b
        ld1             {v6.16b, v7.16b}, [x10], x2
        uaddlp          v2.8h,   v2.16b
        uaddlp          v3.8h,   v3.16b
        uaddlp          v4.8h,   v4.16b
        uaddlp          v5.8h,   v5.16b
        uaddlp          v6.8h,   v6.16b
        uaddlp          v7.8h,   v7.16b
        add             v0.8h,   v0.8h,   v2.8h
        add             v1.8h,   v1.8h,   v3.8h
        add             v4.8h,   v4.8h,   v6.8h
        add             v5.8h,   v5.8h,   v7.8h
        shl             v0.8h,   v0.8h,   #1
        shl             v1.8h,   v1.8h,   #1
        shl             v2.8h,   v4.8h,   #1
        shl             v3.8h,   v5.8h,   #1
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_420_w16_wpad1):
1:      // Copy and subsample input, padding 4
        ldr             d1,  [x1,  #16]
        ld1             {v0.16b}, [x1],  x2
        ldr             d3,  [x10, #16]
        ld1             {v2.16b}, [x10], x2
        uaddlp          v1.4h,   v1.8b
        ldr             d5,  [x1,  #16]
        uaddlp          v0.8h,   v0.16b
        ld1             {v4.16b}, [x1],  x2
        uaddlp          v3.4h,   v3.8b
        ldr             d7,  [x10, #16]
        uaddlp          v2.8h,   v2.16b
        ld1             {v6.16b}, [x10], x2
        uaddlp          v5.4h,   v5.8b
        uaddlp          v4.8h,   v4.16b
        uaddlp          v7.4h,   v7.8b
        uaddlp          v6.8h,   v6.16b
        add             v1.4h,   v1.4h,   v3.4h
        add             v0.8h,   v0.8h,   v2.8h
        add             v5.4h,   v5.4h,   v7.4h
        add             v4.8h,   v4.8h,   v6.8h
        shl             v1.4h,   v1.4h,   #1
        shl             v0.8h,   v0.8h,   #1
        shl             v3.4h,   v5.4h,   #1
        shl             v2.8h,   v4.8h,   #1
        dup             v4.4h,   v1.h[3]
        dup             v5.4h,   v3.h[3]
        trn1            v1.2d,   v1.2d,   v4.2d
        trn1            v3.2d,   v3.2d,   v5.2d
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_420_w16_wpad2):
1:      // Copy and subsample input, padding 8
        ld1             {v0.16b}, [x1],  x2
        ld1             {v2.16b}, [x10], x2
        ld1             {v4.16b}, [x1],  x2
        uaddlp          v0.8h,   v0.16b
        ld1             {v6.16b}, [x10], x2
        uaddlp          v2.8h,   v2.16b
        uaddlp          v4.8h,   v4.16b
        uaddlp          v6.8h,   v6.16b
        add             v0.8h,   v0.8h,   v2.8h
        add             v4.8h,   v4.8h,   v6.8h
        shl             v0.8h,   v0.8h,   #1
        shl             v2.8h,   v4.8h,   #1
        dup             v1.8h,   v0.h[7]
        dup             v3.8h,   v2.h[7]
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_420_w16_wpad3):
1:      // Copy and subsample input, padding 12
        ld1             {v0.8b}, [x1],  x2
        ld1             {v2.8b}, [x10], x2
        ld1             {v4.8b}, [x1],  x2
        uaddlp          v0.4h,   v0.8b
        ld1             {v6.8b}, [x10], x2
        uaddlp          v2.4h,   v2.8b
        uaddlp          v4.4h,   v4.8b
        uaddlp          v6.4h,   v6.8b
        add             v0.4h,   v0.4h,   v2.4h
        add             v4.4h,   v4.4h,   v6.4h
        shl             v0.4h,   v0.4h,   #1
        shl             v2.4h,   v4.4h,   #1
        dup             v1.8h,   v0.h[3]
        dup             v3.8h,   v2.h[3]
        trn1            v0.2d,   v0.2d,   v1.2d
        trn1            v2.2d,   v2.2d,   v3.2d
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_420_w16_hpad):
        cbz             w4,  3f
2:      // Vertical padding (h_pad > 0)
        subs            w4,  w4,  #4
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            2b
3:

        // Double the height and reuse the w8 summing/subtracting
        lsl             w6,  w6,  #1
        lsl             w9,  w9,  #1
        b               L(ipred_cfl_ac_420_w8_calc_subtract_dc)

L(ipred_cfl_ac_420_tbl):
        .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16)
        .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8)
        .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w4)
        .hword 0

L(ipred_cfl_ac_420_w16_tbl):
        .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad0)
        .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1)
        .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2)
        .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3)
endfunc

// void cfl_ac_422_neon(int16_t *const ac, const pixel *const ypx,
//                      const ptrdiff_t stride, const int w_pad,
//                      const int h_pad, const int cw, const int ch);
function ipred_cfl_ac_422_neon, export=1
        clz             w8,  w5
        lsl             w4,  w4,  #2
        adr             x7,  L(ipred_cfl_ac_422_tbl)
        sub             w8,  w8,  #27
        ldrh            w8,  [x7, w8, uxtw #1]
        sub             x7,  x7,  w8, uxtw
        sub             w8,  w6,  w4         // height - h_pad
        rbit            w9,  w5              // rbit(width)
        rbit            w10, w6              // rbit(height)
        clz             w9,  w9              // ctz(width)
        clz             w10, w10             // ctz(height)
        add             w9,  w9,  w10        // log2sz
        movi            v16.4s,  #1
        add             x10, x1,  x2
        lsl             x2,  x2,  #1
        dup             v17.4s,  w9
        sshl            v16.4s,  v16.4s,  v17.4s // 1 << log2sz
        neg             v17.4s,  v17.4s          // -log2sz
        ushr            v16.4s,  v16.4s,  #1     // 1 << (log2sz - 1)
        mov             w9,  w6
        br              x7

L(ipred_cfl_ac_422_w4):
1:      // Copy and subsample input
        ld1             {v0.8b},   [x1],  x2
        ld1             {v0.d}[1], [x10], x2
        ld1             {v1.8b},   [x1],  x2
        ld1             {v1.d}[1], [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v1.8h,   v1.16b
        shl             v0.8h,   v0.8h,   #2
        shl             v1.8h,   v1.8h,   #2
        subs            w8,  w8,  #4
        st1             {v0.8h, v1.8h}, [x0], #32
        b.gt            1b
        trn2            v0.2d,   v1.2d,   v1.2d
        trn2            v1.2d,   v1.2d,   v1.2d
        b               L(ipred_cfl_ac_420_w4_hpad)

L(ipred_cfl_ac_422_w8):
        cbnz            w3,  L(ipred_cfl_ac_422_w8_wpad)
1:      // Copy and subsample input, without padding
        ld1             {v0.16b}, [x1],  x2
        ld1             {v1.16b}, [x10], x2
        ld1             {v2.16b}, [x1],  x2
        uaddlp          v0.8h,   v0.16b
        ld1             {v3.16b}, [x10], x2
        uaddlp          v1.8h,   v1.16b
        uaddlp          v2.8h,   v2.16b
        uaddlp          v3.8h,   v3.16b
        shl             v0.8h,   v0.8h,   #2
        shl             v1.8h,   v1.8h,   #2
        shl             v2.8h,   v2.8h,   #2
        shl             v3.8h,   v3.8h,   #2
        subs            w8,  w8,  #4
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v3.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w8_hpad)

L(ipred_cfl_ac_422_w8_wpad):
1:      // Copy and subsample input, padding 4
        ld1             {v0.8b},   [x1],  x2
        ld1             {v0.d}[1], [x10], x2
        ld1             {v2.8b},   [x1],  x2
        ld1             {v2.d}[1], [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v2.8h,   v2.16b
        shl             v0.8h,   v0.8h,   #2
        shl             v2.8h,   v2.8h,   #2
        dup             v4.4h,   v0.h[3]
        dup             v5.8h,   v0.h[7]
        dup             v6.4h,   v2.h[3]
        dup             v7.8h,   v2.h[7]
        trn2            v1.2d,   v0.2d,   v5.2d
        trn1            v0.2d,   v0.2d,   v4.2d
        trn2            v3.2d,   v2.2d,   v7.2d
        trn1            v2.2d,   v2.2d,   v6.2d
        subs            w8,  w8,  #4
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v3.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w8_hpad)

L(ipred_cfl_ac_422_w16):
        adr             x7,  L(ipred_cfl_ac_422_w16_tbl)
        ldrh            w3,  [x7, w3, uxtw #1]
        sub             x7,  x7,  w3, uxtw
        br              x7

L(ipred_cfl_ac_422_w16_wpad0):
1:      // Copy and subsample input, without padding
        ld1             {v0.16b, v1.16b}, [x1],  x2
        ld1             {v2.16b, v3.16b}, [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v1.8h,   v1.16b
        uaddlp          v2.8h,   v2.16b
        uaddlp          v3.8h,   v3.16b
        shl             v0.8h,   v0.8h,   #2
        shl             v1.8h,   v1.8h,   #2
        shl             v2.8h,   v2.8h,   #2
        shl             v3.8h,   v3.8h,   #2
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_422_w16_wpad1):
1:      // Copy and subsample input, padding 4
        ldr             d1,  [x1,  #16]
        ld1             {v0.16b}, [x1],  x2
        ldr             d3,  [x10, #16]
        ld1             {v2.16b}, [x10], x2
        uaddlp          v1.4h,   v1.8b
        uaddlp          v0.8h,   v0.16b
        uaddlp          v3.4h,   v3.8b
        uaddlp          v2.8h,   v2.16b
        shl             v1.4h,   v1.4h,   #2
        shl             v0.8h,   v0.8h,   #2
        shl             v3.4h,   v3.4h,   #2
        shl             v2.8h,   v2.8h,   #2
        dup             v4.4h,   v1.h[3]
        dup             v5.4h,   v3.h[3]
        trn1            v1.2d,   v1.2d,   v4.2d
        trn1            v3.2d,   v3.2d,   v5.2d
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_422_w16_wpad2):
1:      // Copy and subsample input, padding 8
        ld1             {v0.16b}, [x1],  x2
        ld1             {v2.16b}, [x10], x2
        uaddlp          v0.8h,   v0.16b
        uaddlp          v2.8h,   v2.16b
        shl             v0.8h,   v0.8h,   #2
        shl             v2.8h,   v2.8h,   #2
        dup             v1.8h,   v0.h[7]
        dup             v3.8h,   v2.h[7]
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_422_w16_wpad3):
1:      // Copy and subsample input, padding 12
        ld1             {v0.8b}, [x1],  x2
        ld1             {v2.8b}, [x10], x2
        uaddlp          v0.4h,   v0.8b
        uaddlp          v2.4h,   v2.8b
        shl             v0.4h,   v0.4h,   #2
        shl             v2.4h,   v2.4h,   #2
        dup             v1.8h,   v0.h[3]
        dup             v3.8h,   v2.h[3]
        trn1            v0.2d,   v0.2d,   v1.2d
        trn1            v2.2d,   v2.2d,   v3.2d
        subs            w8,  w8,  #2
        st1             {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
        b.gt            1b
        mov             v0.16b,  v2.16b
        mov             v1.16b,  v3.16b
        b               L(ipred_cfl_ac_420_w16_hpad)

L(ipred_cfl_ac_422_tbl):
        .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w16)
        .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w8)
        .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w4)
        .hword 0

L(ipred_cfl_ac_422_w16_tbl):
        .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0)
        .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1)
        .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2)
        .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3)
endfunc