/*
 * Copyright © 2018, VideoLAN and dav1d authors
 * Copyright © 2020, Martin Storsjo
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

// depending on how many pixels need to be stored, returns:
// x14 = (1 << 0) : 0 pixels
// x14 = (1 << 4) : inner 4 pixels
// x14 = (1 << 6) : inner 6 pixels
// x14 = 0 : all pixels
.macro loop_filter wd
function lpf_8_wd\wd\()_neon
        uabd            v0.8h,   v22.8h,  v23.8h  // abs(p1 - p0)
        uabd            v1.8h,   v25.8h,  v24.8h  // abs(q1 - q0)
        uabd            v2.8h,   v23.8h,  v24.8h  // abs(p0 - q0)
        uabd            v3.8h,   v22.8h,  v25.8h  // abs(p1 - q1)
.if \wd >= 6
        uabd            v4.8h,   v21.8h,  v22.8h  // abs(p2 - p1)
        uabd            v5.8h,   v26.8h,  v25.8h  // abs(q2 - q1)
.endif
.if \wd >= 8
        uabd            v6.8h,   v20.8h,  v21.8h  // abs(p3 - p2)
        uabd            v7.8h,   v27.8h,  v26.8h  // abs(q3 - q3)
.endif
.if \wd >= 6
        umax            v4.8h,   v4.8h,   v5.8h
.endif
        uqadd           v2.8h,   v2.8h,   v2.8h   // abs(p0 - q0) * 2
.if \wd >= 8
        umax            v6.8h,   v6.8h,   v7.8h
.endif
        ushr            v3.8h,   v3.8h,   #1
.if \wd >= 8
        umax            v4.8h,   v4.8h,   v6.8h
.endif
.if \wd >= 6
        and             v4.16b,  v4.16b,  v14.16b
.endif
        umax            v0.8h,   v0.8h,   v1.8h   // max(abs(p1 - p0), abs(q1 - q0))
        uqadd           v2.8h,   v2.8h,   v3.8h   // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1
.if \wd >= 6
        umax            v4.8h,   v0.8h,   v4.8h
        cmhs            v1.8h,   v11.8h,  v4.8h   // max(abs(p1 - p0), abs(q1 - q0), abs(), abs(), ...) <= I
.else
        cmhs            v1.8h,   v11.8h,  v0.8h   // max(abs(p1 - p0), abs(q1 - q0)) <= I
.endif
        cmhs            v2.8h,   v10.8h,  v2.8h   // abs(p0 - q0) * 2 + abs(p1 - q1) >> 1 <= E
        and             v1.16b,  v1.16b,  v2.16b  // fm
        and             v1.16b,  v1.16b,  v13.16b // fm && wd >= 4
.if \wd >= 6
        and             v14.16b, v14.16b, v1.16b  // fm && wd > 4
.endif
.if \wd >= 16
        and             v15.16b, v15.16b, v1.16b  // fm && wd == 16
.endif

        mov             x16, v1.d[0]
        mov             x17, v1.d[1]
        adds            x16, x16, x17
        b.ne            9f                        // if (!fm || wd < 4) return;
        mov             x14, #(1 << 0)
        ret
9:
.if \wd >= 6
        movi            v10.8h,  #1
        uabd            v2.8h,   v21.8h,  v23.8h  // abs(p2 - p0)
        uabd            v3.8h,   v22.8h,  v23.8h  // abs(p1 - p0)
        uabd            v4.8h,   v25.8h,  v24.8h  // abs(q1 - q0)
        uabd            v5.8h,   v26.8h,  v24.8h  // abs(q2 - q0)
        dup             v9.8h,   w9               // bitdepth_min_8
.if \wd >= 8
        uabd            v6.8h,   v20.8h,  v23.8h  // abs(p3 - p0)
        uabd            v7.8h,   v27.8h,  v24.8h  // abs(q3 - q0)
.endif
        umax            v2.8h,   v2.8h,   v3.8h
        umax            v4.8h,   v4.8h,   v5.8h
.if \wd >= 8
        umax            v6.8h,   v6.8h,   v7.8h
.endif
        umax            v2.8h,   v2.8h,   v4.8h
        ushl            v10.8h,  v10.8h,  v9.8h   // F = 1 << bitdepth_min_8
.if \wd >= 8
        umax            v2.8h,   v2.8h,   v6.8h
.endif

.if \wd == 16
        uabd            v3.8h,   v17.8h,  v23.8h  // abs(p6 - p0)
        uabd            v4.8h,   v18.8h,  v23.8h  // abs(p5 - p0)
        uabd            v5.8h,   v19.8h,  v23.8h  // abs(p4 - p0)
.endif
        cmhs            v2.8h,   v10.8h,  v2.8h   // flat8in
.if \wd == 16
        uabd            v6.8h,   v28.8h,  v24.8h  // abs(q4 - q0)
        uabd            v7.8h,   v29.8h,  v24.8h  // abs(q5 - q0)
        uabd            v8.8h,   v30.8h,  v24.8h  // abs(q6 - q0)
.endif
        and             v14.16b, v2.16b,  v14.16b // flat8in && fm && wd > 4
        bic             v1.16b,  v1.16b,  v14.16b // fm && wd >= 4 && !flat8in
.if \wd == 16
        umax            v3.8h,   v3.8h,   v4.8h
        umax            v5.8h,   v5.8h,   v6.8h
.endif
        mov             x16, v1.d[0]
        mov             x17, v1.d[1]
.if \wd == 16
        umax            v7.8h,   v7.8h,   v8.8h
        umax            v3.8h,   v3.8h,   v5.8h
        umax            v3.8h,   v3.8h,   v7.8h
        cmhs            v3.8h,   v10.8h,  v3.8h   // flat8out
.endif
        adds            x16, x16, x17
.if \wd == 16
        and             v15.16b, v15.16b, v3.16b  // flat8out && fm && wd == 16
        and             v15.16b, v15.16b, v14.16b // flat8out && flat8in && fm && wd == 16
        bic             v14.16b, v14.16b, v15.16b // flat8in && fm && wd >= 4 && !flat8out
.endif
        b.eq            1f                        // skip wd == 4 case
.endif

        dup             v3.8h,   w8               // bitdepth_max
        sub             v2.8h,   v22.8h,  v25.8h  // p1 - q1
        ushr            v3.8h,   v3.8h,   #1      // 128 << bitdepth_min_8 - 1
        cmhi            v0.8h,   v0.8h,   v12.8h  // hev
        not             v9.16b,  v3.16b           // - 128 * (1 << bitdepth_min_8)
        smin            v2.8h,   v2.8h,   v3.8h   // iclip_diff(p1 - q1)
        smax            v2.8h,   v2.8h,   v9.8h   // iclip_diff(p1 - q1)
        and             v4.16b,  v2.16b,  v0.16b  // if (hev) iclip_diff(p1 - q1)
        sub             v2.8h,   v24.8h,  v23.8h
        movi            v5.8h,   #3
        bic             v0.16b,  v1.16b,  v0.16b  // (fm && wd >= 4 && !hev)
        mul             v2.8h,   v2.8h,   v5.8h
        movi            v6.8h,   #4
        add             v2.8h,   v2.8h,   v4.8h
        smin            v2.8h,   v2.8h,   v3.8h   // f = iclip_diff()
        smax            v2.8h,   v2.8h,   v9.8h   // f = iclip_diff()
        sqadd           v4.8h,   v6.8h,   v2.8h   // f + 4
        sqadd           v5.8h,   v5.8h,   v2.8h   // f + 3
        smin            v4.8h,   v4.8h,   v3.8h   // imin(f + 4, 128 << bitdepth_min_8 - 1)
        smin            v5.8h,   v5.8h,   v3.8h   // imin(f + 3, 128 << bitdepth_min_8 - 1)
        sshr            v4.8h,   v4.8h,   #3      // f1
        sshr            v5.8h,   v5.8h,   #3      // f2
        movi            v9.8h,   #0
        dup             v3.8h,   w8               // bitdepth_max
        sqadd           v2.8h,   v23.8h,  v5.8h   // p0 + f2
        sqsub           v6.8h,   v24.8h,  v4.8h   // q0 - f1
        srshr           v4.8h,   v4.8h,   #1      // (f1 + 1) >> 1
        smin            v2.8h,   v2.8h,   v3.8h   // out p0 = iclip_pixel()
        smin            v6.8h,   v6.8h,   v3.8h   // out q0 = iclip_pixel()
        smax            v2.8h,   v2.8h,   v9.8h   // out p0 = iclip_pixel()
        smax            v6.8h,   v6.8h,   v9.8h   // out q0 = iclip_pixel()
        bit             v23.16b, v2.16b,  v1.16b  // if (fm && wd >= 4)
        bit             v24.16b, v6.16b,  v1.16b  // if (fm && wd >= 4)
        sqadd           v2.8h,   v22.8h,  v4.8h   // p1 + f
        sqsub           v6.8h,   v25.8h,  v4.8h   // q1 - f
        smin            v2.8h,   v2.8h,   v3.8h   // out p1 = iclip_pixel()
        smin            v6.8h,   v6.8h,   v3.8h   // out q1 = iclip_pixel()
        smax            v2.8h,   v2.8h,   v9.8h   // out p1 = iclip_pixel()
        smax            v6.8h,   v6.8h,   v9.8h   // out q1 = iclip_pixel()
        bit             v22.16b, v2.16b,  v0.16b  // if (fm && wd >= 4 && !hev)
        bit             v25.16b, v6.16b,  v0.16b  // if (fm && wd >= 4 && !hev)
1:

.if \wd == 6
        mov             x16, v14.d[0]
        mov             x17, v14.d[1]
        adds            x16, x16, x17
        b.eq            2f                        // skip if there's no flat8in

        add             v0.8h,   v21.8h,  v21.8h  // p2 * 2
        add             v2.8h,   v21.8h,  v22.8h  // p2 + p1
        add             v4.8h,   v22.8h,  v23.8h  // p1 + p0
        add             v6.8h,   v23.8h,  v24.8h  // p0 + q0
        add             v8.8h,   v0.8h,   v2.8h
        add             v10.8h,  v4.8h,   v6.8h
        add             v12.8h,  v24.8h,  v25.8h  // q0 + q1
        add             v8.8h,   v8.8h,   v10.8h
        sub             v12.8h,  v12.8h,  v0.8h
        add             v10.8h,  v25.8h,  v26.8h  // q1 + q2
        urshr           v0.8h,   v8.8h,   #3      // out p1

        add             v8.8h,   v8.8h,   v12.8h
        sub             v10.8h,  v10.8h,  v2.8h
        add             v12.8h,  v26.8h,  v26.8h  // q2 + q2
        urshr           v1.8h,   v8.8h,   #3      // out p0

        add             v8.8h,   v8.8h,   v10.8h
        sub             v12.8h,  v12.8h,  v4.8h
        urshr           v2.8h,   v8.8h,   #3      // out q0

        bit             v22.16b, v0.16b,  v14.16b // p1 if (flat8in)
        add             v8.8h,   v8.8h,   v12.8h
        bit             v23.16b, v1.16b,  v14.16b // p0 if (flat8in)
        urshr           v3.8h,   v8.8h,   #3      // out q1
        bit             v24.16b, v2.16b,  v14.16b // q0 if (flat8in)
        bit             v25.16b, v3.16b,  v14.16b // q1 if (flat8in)
.elseif \wd >= 8
        mov             x16, v14.d[0]
        mov             x17, v14.d[1]
        adds            x16, x16, x17
.if \wd == 8
        b.eq            8f                        // skip if there's no flat8in
.else
        b.eq            2f                        // skip if there's no flat8in
.endif

        add             v0.8h,   v20.8h,  v21.8h  // p3 + p2
        add             v2.8h,   v22.8h,  v25.8h  // p1 + q1
        add             v4.8h,   v20.8h,  v22.8h  // p3 + p1
        add             v6.8h,   v23.8h,  v26.8h  // p0 + q2
        add             v8.8h,   v0.8h,   v0.8h   // 2 * (p3 + p2)
        add             v9.8h,   v23.8h,  v24.8h  // p0 + q0
        add             v8.8h,   v8.8h,   v4.8h   // + p3 + p1
        sub             v2.8h,   v2.8h,   v0.8h   // p1 + q1 - p3 - p2
        add             v8.8h,   v8.8h,   v9.8h   // + p0 + q0
        sub             v6.8h,   v6.8h,   v4.8h   // p0 + q2 - p3 - p1
        urshr           v10.8h,  v8.8h,   #3      // out p2

        add             v8.8h,   v8.8h,   v2.8h
        add             v0.8h,   v20.8h,  v23.8h  // p3 + p0
        add             v2.8h,   v24.8h,  v27.8h  // q0 + q3
        urshr           v11.8h,  v8.8h,   #3      // out p1

        add             v8.8h,   v8.8h,   v6.8h
        sub             v2.8h,   v2.8h,   v0.8h   // q0 + q3 - p3 - p0
        add             v4.8h,   v21.8h,  v24.8h  // p2 + q0
        add             v6.8h,   v25.8h,  v27.8h  // q1 + q3
        urshr           v12.8h,  v8.8h,   #3      // out p0

        add             v8.8h,   v8.8h,   v2.8h
        sub             v6.8h,   v6.8h,   v4.8h   // q1 + q3 - p2 - q0
        add             v0.8h,   v22.8h,  v25.8h  // p1 + q1
        add             v2.8h,   v26.8h,  v27.8h  // q2 + q3
        urshr           v13.8h,  v8.8h,   #3      // out q0

        add             v8.8h,   v8.8h,   v6.8h
        sub             v2.8h,   v2.8h,   v0.8h   // q2 + q3 - p1 - q1
        urshr           v0.8h,   v8.8h,   #3      // out q1

        add             v8.8h,   v8.8h,   v2.8h

        bit             v21.16b, v10.16b, v14.16b
        bit             v22.16b, v11.16b, v14.16b
        bit             v23.16b, v12.16b, v14.16b
        urshr           v1.8h,   v8.8h,   #3      // out q2
        bit             v24.16b, v13.16b, v14.16b
        bit             v25.16b, v0.16b,  v14.16b
        bit             v26.16b, v1.16b,  v14.16b
.endif
2:
.if \wd == 16
        mov             x16, v15.d[0]
        mov             x17, v15.d[1]
        adds            x16, x16, x17
        b.ne            1f                        // check if flat8out is needed
        mov             x16, v14.d[0]
        mov             x17, v14.d[1]
        adds            x16, x16, x17
        b.eq            8f                        // if there was no flat8in, just write the inner 4 pixels
        b               7f                        // if flat8in was used, write the inner 6 pixels
1:

        add             v2.8h,   v17.8h,  v17.8h  // p6 + p6
        add             v4.8h,   v17.8h,  v18.8h  // p6 + p5
        add             v6.8h,   v17.8h,  v19.8h  // p6 + p4
        add             v8.8h,   v17.8h,  v20.8h  // p6 + p3
        add             v12.8h,  v2.8h,   v4.8h
        add             v10.8h,  v6.8h,   v8.8h
        add             v6.8h,   v17.8h,  v21.8h  // p6 + p2
        add             v12.8h,  v12.8h,  v10.8h
        add             v8.8h,   v17.8h,  v22.8h  // p6 + p1
        add             v10.8h,  v18.8h,  v23.8h  // p5 + p0
        add             v6.8h,   v6.8h,   v8.8h
        add             v8.8h,   v19.8h,  v24.8h  // p4 + q0
        add             v12.8h,  v12.8h,  v6.8h
        add             v10.8h,  v10.8h,  v8.8h
        add             v6.8h,   v20.8h,  v25.8h  // p3 + q1
        add             v12.8h,  v12.8h,  v10.8h
        sub             v6.8h,   v6.8h,   v2.8h
        add             v2.8h,   v21.8h,  v26.8h  // p2 + q2
        urshr           v0.8h,   v12.8h,  #4      // out p5
        add             v12.8h,  v12.8h,  v6.8h   // - (p6 + p6) + (p3 + q1)
        sub             v2.8h,   v2.8h,   v4.8h
        add             v4.8h,   v22.8h,  v27.8h  // p1 + q3
        add             v6.8h,   v17.8h,  v19.8h  // p6 + p4
        urshr           v1.8h,   v12.8h,  #4      // out p4
        add             v12.8h,  v12.8h,  v2.8h   // - (p6 + p5) + (p2 + q2)
        sub             v4.8h,   v4.8h,   v6.8h
        add             v6.8h,   v23.8h,  v28.8h  // p0 + q4
        add             v8.8h,   v17.8h,  v20.8h  // p6 + p3
        urshr           v2.8h,   v12.8h,  #4      // out p3
        add             v12.8h,  v12.8h,  v4.8h   // - (p6 + p4) + (p1 + q3)
        sub             v6.8h,   v6.8h,   v8.8h
        add             v8.8h,   v24.8h,  v29.8h  // q0 + q5
        add             v4.8h,   v17.8h,  v21.8h  // p6 + p2
        urshr           v3.8h,   v12.8h,  #4      // out p2
        add             v12.8h,  v12.8h,  v6.8h   // - (p6 + p3) + (p0 + q4)
        sub             v8.8h,   v8.8h,   v4.8h
        add             v6.8h,   v25.8h,  v30.8h  // q1 + q6
        add             v10.8h,  v17.8h,  v22.8h  // p6 + p1
        urshr           v4.8h,   v12.8h,  #4      // out p1
        add             v12.8h,  v12.8h,  v8.8h   // - (p6 + p2) + (q0 + q5)
        sub             v6.8h,   v6.8h,   v10.8h
        add             v8.8h,   v26.8h,  v30.8h  // q2 + q6
        bif             v0.16b,  v18.16b, v15.16b // out p5
        add             v10.8h,  v18.8h,  v23.8h  // p5 + p0
        urshr           v5.8h,   v12.8h,  #4      // out p0
        add             v12.8h,  v12.8h,  v6.8h   // - (p6 + p1) + (q1 + q6)
        sub             v8.8h,   v8.8h,   v10.8h
        add             v10.8h,  v27.8h,  v30.8h  // q3 + q6
        bif             v1.16b,  v19.16b, v15.16b // out p4
        add             v18.8h,  v19.8h,  v24.8h  // p4 + q0
        urshr           v6.8h,   v12.8h,  #4      // out q0
        add             v12.8h,  v12.8h,  v8.8h   // - (p5 + p0) + (q2 + q6)
        sub             v10.8h,  v10.8h,  v18.8h
        add             v8.8h,   v28.8h,  v30.8h  // q4 + q6
        bif             v2.16b,  v20.16b, v15.16b // out p3
        add             v18.8h,  v20.8h,  v25.8h  // p3 + q1
        urshr           v7.8h,   v12.8h,  #4      // out q1
        add             v12.8h,  v12.8h,  v10.8h  // - (p4 + q0) + (q3 + q6)
        sub             v18.8h,  v8.8h,   v18.8h
        add             v10.8h,  v29.8h,  v30.8h  // q5 + q6
        bif             v3.16b,  v21.16b, v15.16b // out p2
        add             v20.8h,  v21.8h,  v26.8h  // p2 + q2
        urshr           v8.8h,   v12.8h,  #4      // out q2
        add             v12.8h,  v12.8h,  v18.8h  // - (p3 + q1) + (q4 + q6)
        sub             v10.8h,  v10.8h,  v20.8h
        add             v18.8h,  v30.8h,  v30.8h  // q6 + q6
        bif             v4.16b,  v22.16b, v15.16b // out p1
        add             v20.8h,  v22.8h,  v27.8h  // p1 + q3
        urshr           v9.8h,   v12.8h,  #4      // out q3
        add             v12.8h,  v12.8h,  v10.8h  // - (p2 + q2) + (q5 + q6)
        sub             v18.8h,  v18.8h,  v20.8h
        bif             v5.16b,  v23.16b, v15.16b // out p0
        urshr           v10.8h,  v12.8h,  #4      // out q4
        add             v12.8h,  v12.8h,  v18.8h  // - (p1 + q3) + (q6 + q6)
        urshr           v11.8h,  v12.8h,  #4      // out q5
        bif             v6.16b,  v24.16b, v15.16b // out q0
        bif             v7.16b,  v25.16b, v15.16b // out q1
        bif             v8.16b,  v26.16b, v15.16b // out q2
        bif             v9.16b,  v27.16b, v15.16b // out q3
        bif             v10.16b, v28.16b, v15.16b // out q4
        bif             v11.16b, v29.16b, v15.16b // out q5
.endif

        mov             x14, #0
        ret
.if \wd == 16
7:
        // Return to a shorter epilogue, writing only the inner 6 pixels
        mov             x14, #(1 << 6)
        ret
.endif
.if \wd >= 8
8:
        // Return to a shorter epilogue, writing only the inner 4 pixels
        mov             x14, #(1 << 4)
        ret
.endif
endfunc
.endm

loop_filter 16
loop_filter 8
loop_filter 6
loop_filter 4

.macro lpf_8_wd16
        bl              lpf_8_wd16_neon
        cbz             x14, 1f
        tbnz            x14, #6, 7f
        tbnz            x14, #4, 8f
        ret             x15
1:
.endm

.macro lpf_8_wd8
        bl              lpf_8_wd8_neon
        cbz             x14, 1f
        tbnz            x14, #4, 8f
        ret             x15
1:
.endm

.macro lpf_8_wd6
        bl              lpf_8_wd6_neon
        cbz             x14, 1f
        ret             x15
1:
.endm

.macro lpf_8_wd4
        bl              lpf_8_wd4_neon
        cbz             x14, 1f
        ret             x15
1:
.endm

function lpf_v_4_8_neon
        mov             x15, x30
        sub             x16, x0,  x1, lsl #1
        ld1             {v22.8h}, [x16], x1 // p1
        ld1             {v24.8h}, [x0],  x1 // q0
        ld1             {v23.8h}, [x16], x1 // p0
        ld1             {v25.8h}, [x0],  x1 // q1
        sub             x0,  x0,  x1, lsl #1

        lpf_8_wd4

        sub             x16, x0,  x1, lsl #1
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v25.8h}, [x0],  x1 // q1
        sub             x0,  x0,  x1, lsl #1
        ret             x15
endfunc

function lpf_h_4_8_neon
        mov             x15, x30
        sub             x16, x0,  #4
        add             x0,  x16, x1, lsl #2
        ld1             {v22.d}[0], [x16], x1
        ld1             {v22.d}[1], [x0],  x1
        ld1             {v23.d}[0], [x16], x1
        ld1             {v23.d}[1], [x0],  x1
        ld1             {v24.d}[0], [x16], x1
        ld1             {v24.d}[1], [x0],  x1
        ld1             {v25.d}[0], [x16], x1
        ld1             {v25.d}[1], [x0],  x1
        add             x0,  x0,  #4

        transpose_4x8h  v22, v23, v24, v25, v26, v27, v28, v29

        lpf_8_wd4

        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #4
        transpose_4x8h  v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v22.d}[0], [x16], x1
        st1             {v22.d}[1], [x0],  x1
        st1             {v23.d}[0], [x16], x1
        st1             {v23.d}[1], [x0],  x1
        st1             {v24.d}[0], [x16], x1
        st1             {v24.d}[1], [x0],  x1
        st1             {v25.d}[0], [x16], x1
        st1             {v25.d}[1], [x0],  x1
        add             x0,  x0,  #4
        ret             x15
endfunc

function lpf_v_6_8_neon
        mov             x15, x30
        sub             x16, x0,  x1, lsl #1
        sub             x16, x16, x1
        ld1             {v21.8h}, [x16], x1 // p2
        ld1             {v24.8h}, [x0],  x1 // q0
        ld1             {v22.8h}, [x16], x1 // p1
        ld1             {v25.8h}, [x0],  x1 // q1
        ld1             {v23.8h}, [x16], x1 // p0
        ld1             {v26.8h}, [x0],  x1 // q2
        sub             x0,  x0,  x1, lsl #1
        sub             x0,  x0,  x1

        lpf_8_wd6

        sub             x16, x0,  x1, lsl #1
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v25.8h}, [x0],  x1 // q1
        sub             x0,  x0,  x1, lsl #1
        ret             x15
endfunc

function lpf_h_6_8_neon
        mov             x15, x30
        sub             x16, x0,  #8
        add             x0,  x16, x1, lsl #2
        ld1             {v20.8h}, [x16], x1
        ld1             {v24.8h}, [x0],  x1
        ld1             {v21.8h}, [x16], x1
        ld1             {v25.8h}, [x0],  x1
        ld1             {v22.8h}, [x16], x1
        ld1             {v26.8h}, [x0],  x1
        ld1             {v23.8h}, [x16], x1
        ld1             {v27.8h}, [x0],  x1
        add             x0,  x0,  #8

        transpose_8x8h  v20, v21, v22, v23, v24, v25, v26, v27, v28, v29

        lpf_8_wd6

        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #4
        transpose_4x8h  v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v22.d}[0], [x16], x1
        st1             {v22.d}[1], [x0],  x1
        st1             {v23.d}[0], [x16], x1
        st1             {v23.d}[1], [x0],  x1
        st1             {v24.d}[0], [x16], x1
        st1             {v24.d}[1], [x0],  x1
        st1             {v25.d}[0], [x16], x1
        st1             {v25.d}[1], [x0],  x1
        add             x0,  x0,  #4
        ret             x15
endfunc

function lpf_v_8_8_neon
        mov             x15, x30
        sub             x16, x0,  x1, lsl #2
        ld1             {v20.8h}, [x16], x1 // p3
        ld1             {v24.8h}, [x0],  x1 // q0
        ld1             {v21.8h}, [x16], x1 // p2
        ld1             {v25.8h}, [x0],  x1 // q1
        ld1             {v22.8h}, [x16], x1 // p1
        ld1             {v26.8h}, [x0],  x1 // q2
        ld1             {v23.8h}, [x16], x1 // p0
        ld1             {v27.8h}, [x0],  x1 // q3
        sub             x0,  x0,  x1, lsl #2

        lpf_8_wd8

        sub             x16, x0,  x1, lsl #1
        sub             x16, x16,  x1
        st1             {v21.8h}, [x16], x1 // p2
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v25.8h}, [x0],  x1 // q1
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v26.8h}, [x0],  x1 // q2
        sub             x0,  x0,  x1, lsl #1
        sub             x0,  x0,  x1
        ret             x15

8:
        sub             x16, x0,  x1, lsl #1
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v25.8h}, [x0],  x1 // q1
        sub             x0,  x0,  x1, lsl #1
        ret             x15
endfunc

function lpf_h_8_8_neon
        mov             x15, x30
        sub             x16, x0,  #8
        add             x0,  x16, x1, lsl #2
        ld1             {v20.8h}, [x16], x1
        ld1             {v24.8h}, [x0],  x1
        ld1             {v21.8h}, [x16], x1
        ld1             {v25.8h}, [x0],  x1
        ld1             {v22.8h}, [x16], x1
        ld1             {v26.8h}, [x0],  x1
        ld1             {v23.8h}, [x16], x1
        ld1             {v27.8h}, [x0],  x1
        add             x0,  x0,  #8

        transpose_8x8h  v20, v21, v22, v23, v24, v25, v26, v27, v28, v29

        lpf_8_wd8

        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #8
        transpose_8x8h  v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v20.8h}, [x16], x1
        st1             {v24.8h}, [x0],  x1
        st1             {v21.8h}, [x16], x1
        st1             {v25.8h}, [x0],  x1
        st1             {v22.8h}, [x16], x1
        st1             {v26.8h}, [x0],  x1
        st1             {v23.8h}, [x16], x1
        st1             {v27.8h}, [x0],  x1
        add             x0,  x0,  #8
        ret             x15
8:
        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #4
        transpose_4x8h  v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v22.d}[0], [x16], x1
        st1             {v22.d}[1], [x0],  x1
        st1             {v23.d}[0], [x16], x1
        st1             {v23.d}[1], [x0],  x1
        st1             {v24.d}[0], [x16], x1
        st1             {v24.d}[1], [x0],  x1
        st1             {v25.d}[0], [x16], x1
        st1             {v25.d}[1], [x0],  x1
        add             x0,  x0,  #4
        ret             x15
endfunc

function lpf_v_16_8_neon
        mov             x15, x30

        sub             x16, x0,  x1, lsl #3
        add             x16, x16, x1
        ld1             {v17.8h}, [x16], x1 // p6
        ld1             {v24.8h}, [x0],  x1 // q0
        ld1             {v18.8h}, [x16], x1 // p5
        ld1             {v25.8h}, [x0],  x1 // q1
        ld1             {v19.8h}, [x16], x1 // p4
        ld1             {v26.8h}, [x0],  x1 // q2
        ld1             {v20.8h}, [x16], x1 // p3
        ld1             {v27.8h}, [x0],  x1 // q3
        ld1             {v21.8h}, [x16], x1 // p2
        ld1             {v28.8h}, [x0],  x1 // q4
        ld1             {v22.8h}, [x16], x1 // p1
        ld1             {v29.8h}, [x0],  x1 // q5
        ld1             {v23.8h}, [x16], x1 // p0
        ld1             {v30.8h}, [x0],  x1 // q6
        sub             x0,  x0,  x1, lsl #3
        add             x0,  x0,  x1

        lpf_8_wd16

        sub             x16, x0,  x1, lsl #2
        sub             x16, x16, x1, lsl #1
        st1             {v0.8h},  [x16], x1 // p5
        st1             {v6.8h},  [x0],  x1 // q0
        st1             {v1.8h},  [x16], x1 // p4
        st1             {v7.8h},  [x0],  x1 // q1
        st1             {v2.8h},  [x16], x1 // p3
        st1             {v8.8h},  [x0],  x1 // q2
        st1             {v3.8h},  [x16], x1 // p2
        st1             {v9.8h},  [x0],  x1 // q3
        st1             {v4.8h},  [x16], x1 // p1
        st1             {v10.8h}, [x0],  x1 // q4
        st1             {v5.8h},  [x16], x1 // p0
        st1             {v11.8h}, [x0],  x1 // q5
        sub             x0,  x0,  x1, lsl #2
        sub             x0,  x0,  x1, lsl #1
        ret             x15
7:
        sub             x16, x0,  x1
        sub             x16, x16, x1, lsl #1
        st1             {v21.8h}, [x16], x1 // p2
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v25.8h}, [x0],  x1 // q1
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v26.8h}, [x0],  x1 // q2
        sub             x0,  x0,  x1, lsl #1
        sub             x0,  x0,  x1
        ret             x15

8:
        sub             x16, x0,  x1, lsl #1
        st1             {v22.8h}, [x16], x1 // p1
        st1             {v24.8h}, [x0],  x1 // q0
        st1             {v23.8h}, [x16], x1 // p0
        st1             {v25.8h}, [x0],  x1 // q1
        sub             x0,  x0,  x1, lsl #1
        ret             x15
endfunc

function lpf_h_16_8_neon
        mov             x15, x30
        sub             x16, x0,  #16
        ld1             {v16.8h}, [x16], x1
        ld1             {v24.8h}, [x0],  x1
        ld1             {v17.8h}, [x16], x1
        ld1             {v25.8h}, [x0],  x1
        ld1             {v18.8h}, [x16], x1
        ld1             {v26.8h}, [x0],  x1
        ld1             {v19.8h}, [x16], x1
        ld1             {v27.8h}, [x0],  x1
        ld1             {v20.8h}, [x16], x1
        ld1             {v28.8h}, [x0],  x1
        ld1             {v21.8h}, [x16], x1
        ld1             {v29.8h}, [x0],  x1
        ld1             {v22.8h}, [x16], x1
        ld1             {v30.8h}, [x0],  x1
        ld1             {v23.8h}, [x16], x1
        ld1             {v31.8h}, [x0],  x1

        transpose_8x8h  v16, v17, v18, v19, v20, v21, v22, v23, v0,  v1
        transpose_8x8h  v24, v25, v26, v27, v28, v29, v30, v31, v0,  v1

        lpf_8_wd16

        sub             x0,  x0,  x1, lsl #3
        sub             x16, x0,  #16

        transpose_8x8h  v16, v17, v0,  v1,  v2,  v3,  v4,  v5,  v18, v19
        transpose_8x8h  v6,  v7,  v8,  v9,  v10, v11, v30, v31, v18, v19

        st1             {v16.8h}, [x16], x1
        st1             {v6.8h},  [x0],  x1
        st1             {v17.8h}, [x16], x1
        st1             {v7.8h},  [x0],  x1
        st1             {v0.8h},  [x16], x1
        st1             {v8.8h},  [x0],  x1
        st1             {v1.8h},  [x16], x1
        st1             {v9.8h},  [x0],  x1
        st1             {v2.8h},  [x16], x1
        st1             {v10.8h}, [x0],  x1
        st1             {v3.8h},  [x16], x1
        st1             {v11.8h}, [x0],  x1
        st1             {v4.8h},  [x16], x1
        st1             {v30.8h}, [x0],  x1
        st1             {v5.8h},  [x16], x1
        st1             {v31.8h}, [x0],  x1
        ret             x15

7:
        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #8
        transpose_8x8h  v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v20.8h}, [x16], x1
        st1             {v24.8h}, [x0],  x1
        st1             {v21.8h}, [x16], x1
        st1             {v25.8h}, [x0],  x1
        st1             {v22.8h}, [x16], x1
        st1             {v26.8h}, [x0],  x1
        st1             {v23.8h}, [x16], x1
        st1             {v27.8h}, [x0],  x1
        add             x0,  x0,  #8
        ret             x15
8:
        sub             x16, x0,  x1, lsl #3
        sub             x16, x16, #4
        transpose_4x8h  v22, v23, v24, v25, v26, v27, v28, v29
        add             x0,  x16, x1, lsl #2

        st1             {v22.d}[0], [x16], x1
        st1             {v22.d}[1], [x0],  x1
        st1             {v23.d}[0], [x16], x1
        st1             {v23.d}[1], [x0],  x1
        st1             {v24.d}[0], [x16], x1
        st1             {v24.d}[1], [x0],  x1
        st1             {v25.d}[0], [x16], x1
        st1             {v25.d}[1], [x0],  x1
        add             x0,  x0,  #4
        ret             x15
endfunc

// void dav1d_lpf_v_sb_y_16bpc_neon(pixel *dst, const ptrdiff_t stride,
//                                  const uint32_t *const vmask,
//                                  const uint8_t (*l)[4], ptrdiff_t b4_stride,
//                                  const Av1FilterLUT *lut, const int w,
//                                  const int bitdepth_max)

.macro lpf_func dir, type
function lpf_\dir\()_sb_\type\()_16bpc_neon, export=1
        mov             x11, x30
        mov             w8,  w7  // bitdepth_max
        clz             w9,  w8
        mov             w10, #24
        sub             w9,  w10,  w9 // bitdepth_min_8
        stp             d8,  d9,  [sp, #-0x40]!
        stp             d10, d11, [sp, #0x10]
        stp             d12, d13, [sp, #0x20]
        stp             d14, d15, [sp, #0x30]
        ldp             w6,  w7,  [x2]           // vmask[0], vmask[1]
.ifc \type, y
        ldr             w2,  [x2, #8]            // vmask[2]
.endif
        add             x5,  x5,  #128           // Move to sharp part of lut
.ifc \type, y
        orr             w7,  w7,  w2             // vmask[1] |= vmask[2]
.endif
.ifc \dir, v
        sub             x4,  x3,  x4, lsl #2
.else
        sub             x3,  x3,  #4
        lsl             x4,  x4,  #2
.endif
        orr             w6,  w6,  w7             // vmask[0] |= vmask[1]

1:
        tst             w6,  #0x03
.ifc \dir, v
        ld1             {v0.8b}, [x4], #8
        ld1             {v1.8b}, [x3], #8
.else
        ld2             {v0.s,v1.s}[0], [x3], x4
        ld2             {v0.s,v1.s}[1], [x3], x4
.endif
        b.eq            7f                        // if (!(vm & bits)) continue;

        ld1r            {v5.8b}, [x5]             // sharp[0]
        add             x5,  x5,  #8
        movi            v2.2s,   #0xff
        dup             v13.2s,  w6               // vmask[0]
        dup             v31.8h,  w9               // bitdepth_min_8

        and             v0.8b,   v0.8b,   v2.8b   // Keep only lowest byte in each 32 bit word
        and             v1.8b,   v1.8b,   v2.8b
        cmtst           v3.8b,   v1.8b,   v2.8b   // Check for nonzero values in l[0][0]
        movi            v4.8b,   #1
        ld1r            {v6.8b}, [x5]             // sharp[1]
        sub             x5,  x5,  #8
        bif             v1.8b,   v0.8b,   v3.8b   // if (!l[0][0]) L = l[offset][0]
        cmtst           v2.2s,   v1.2s,   v2.2s   // L != 0
        mul             v1.2s,   v1.2s,   v4.2s   // L
.ifc \type, y
        dup             v15.2s,  w2               // vmask[2]
.endif
        dup             v14.2s,  w7               // vmask[1]
        mov             x16, v2.d[0]
        cmp             x16, #0
        b.eq            7f                        // if (!L) continue;
        neg             v5.8b,   v5.8b            // -sharp[0]
        movrel          x16,  word_12
        ushr            v12.8b,  v1.8b,   #4      // H
        ld1             {v16.2s}, [x16]
        sshl            v3.8b,   v1.8b,   v5.8b   // L >> sharp[0]
.ifc \type, y
        cmtst           v15.2s,  v15.2s,  v16.2s  // if (vmask[2] & bits)
.endif
        movi            v7.8b,   #2
        umin            v3.8b,   v3.8b,   v6.8b   // imin(L >> sharp[0], sharp[1])
        add             v0.8b,   v1.8b,   v7.8b   // L + 2
        umax            v11.8b,  v3.8b,   v4.8b   // imax(imin(), 1) = limit = I
        add             v0.8b,   v0.8b,   v0.8b   // 2*(L + 2)
        cmtst           v14.2s,  v14.2s,  v16.2s  // if (vmask[1] & bits)
        uxtl            v12.8h,  v12.8b
        add             v10.8b,  v0.8b,   v11.8b  // 2*(L + 2) + limit = E
        cmtst           v13.2s,  v13.2s,  v16.2s  // if (vmask[0] & bits)
        uxtl            v11.8h,  v11.8b
        uxtl            v10.8h,  v10.8b
        and             v13.8b,  v13.8b,  v2.8b   // vmask[0] &= L != 0
        sxtl            v14.8h,  v14.8b
        sxtl            v13.8h,  v13.8b
.ifc \type, y
        sxtl            v15.8h,  v15.8b
.endif
        ushl            v12.8h,  v12.8h,  v31.8h
        ushl            v11.8h,  v11.8h,  v31.8h
        ushl            v10.8h,  v10.8h,  v31.8h

.ifc \type, y
        tst             w2,  #0x03
        b.eq            2f
        // wd16
        bl              lpf_\dir\()_16_8_neon
        b               8f
2:
.endif
        tst             w7,  #0x03
        b.eq            3f
.ifc \type, y
        // wd8
        bl              lpf_\dir\()_8_8_neon
.else
        // wd6
        bl              lpf_\dir\()_6_8_neon
.endif
        b               8f
3:
        // wd4
        bl              lpf_\dir\()_4_8_neon
.ifc \dir, h
        b               8f
7:
        // For dir h, the functions above increment x0.
        // If the whole function is skipped, increment it here instead.
        add             x0,  x0,  x1,  lsl #3
.else
7:
.endif
8:
        lsr             w6,  w6,  #2              // vmask[0] >>= 2
        lsr             w7,  w7,  #2              // vmask[1] >>= 2
.ifc \type, y
        lsr             w2,  w2,  #2              // vmask[2] >>= 2
.endif
.ifc \dir, v
        add             x0,  x0,  #16
.else
        // For dir h, x0 is returned incremented
.endif
        cbnz            w6,  1b

        ldp             d14, d15, [sp, #0x30]
        ldp             d12, d13, [sp, #0x20]
        ldp             d10, d11, [sp, #0x10]
        ldp             d8,  d9,  [sp], 0x40
        ret             x11
endfunc
.endm

lpf_func v, y
lpf_func h, y
lpf_func v, uv
lpf_func h, uv

const word_12
        .word 1, 2
endconst