| /* |
| * Copyright © 2018, VideoLAN and dav1d authors |
| * Copyright © 2018, Martin Storsjo |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * |
| * 1. Redistributions of source code must retain the above copyright notice, this |
| * list of conditions and the following disclaimer. |
| * |
| * 2. Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "src/arm/asm.S" |
| |
| #define FILTER_OUT_STRIDE 384 |
| |
| .macro sgr_funcs bpc |
| // void dav1d_sgr_finish_filter1_Xbpc_neon(int16_t *tmp, |
| // const pixel *src, const ptrdiff_t stride, |
| // const int32_t *a, const int16_t *b, |
| // const int w, const int h); |
| function sgr_finish_filter1_\bpc\()bpc_neon, export=1 |
| sub x7, x3, #(4*SUM_STRIDE) |
| add x8, x3, #(4*SUM_STRIDE) |
| sub x9, x4, #(2*SUM_STRIDE) |
| add x10, x4, #(2*SUM_STRIDE) |
| mov x11, #SUM_STRIDE |
| mov x12, #FILTER_OUT_STRIDE |
| add x13, x5, #7 |
| bic x13, x13, #7 // Aligned width |
| .if \bpc == 8 |
| sub x2, x2, x13 |
| .else |
| sub x2, x2, x13, lsl #1 |
| .endif |
| sub x12, x12, x13 |
| sub x11, x11, x13 |
| sub x11, x11, #4 // We read 4 extra elements from a |
| sub x14, x11, #4 // We read 8 extra elements from b |
| mov x13, x5 |
| movi v6.8h, #3 |
| movi v7.4s, #3 |
| 1: |
| ld1 {v0.8h, v1.8h}, [x9], #32 |
| ld1 {v2.8h, v3.8h}, [x4], #32 |
| ld1 {v4.8h, v5.8h}, [x10], #32 |
| ld1 {v16.4s, v17.4s, v18.4s}, [x7], #48 |
| ld1 {v19.4s, v20.4s, v21.4s}, [x3], #48 |
| ld1 {v22.4s, v23.4s, v24.4s}, [x8], #48 |
| |
| 2: |
| subs x5, x5, #8 |
| ext v25.16b, v0.16b, v1.16b, #2 // -stride |
| ext v26.16b, v2.16b, v3.16b, #2 // 0 |
| ext v27.16b, v4.16b, v5.16b, #2 // +stride |
| ext v28.16b, v0.16b, v1.16b, #4 // +1-stride |
| ext v29.16b, v2.16b, v3.16b, #4 // +1 |
| ext v30.16b, v4.16b, v5.16b, #4 // +1+stride |
| add v2.8h, v2.8h, v25.8h // -1, -stride |
| add v26.8h, v26.8h, v27.8h // 0, +stride |
| add v0.8h, v0.8h, v28.8h // -1-stride, +1-stride |
| add v2.8h, v2.8h, v26.8h |
| add v4.8h, v4.8h, v30.8h // -1+stride, +1+stride |
| add v2.8h, v2.8h, v29.8h // +1 |
| add v0.8h, v0.8h, v4.8h |
| |
| ext v25.16b, v16.16b, v17.16b, #4 // -stride |
| ext v26.16b, v17.16b, v18.16b, #4 |
| shl v2.8h, v2.8h, #2 |
| ext v27.16b, v16.16b, v17.16b, #8 // +1-stride |
| ext v28.16b, v17.16b, v18.16b, #8 |
| ext v29.16b, v19.16b, v20.16b, #4 // 0 |
| ext v30.16b, v20.16b, v21.16b, #4 |
| mla v2.8h, v0.8h, v6.8h // * 3 -> a |
| add v25.4s, v25.4s, v19.4s // -stride, -1 |
| add v26.4s, v26.4s, v20.4s |
| add v16.4s, v16.4s, v27.4s // -1-stride, +1-stride |
| add v17.4s, v17.4s, v28.4s |
| ext v27.16b, v19.16b, v20.16b, #8 // +1 |
| ext v28.16b, v20.16b, v21.16b, #8 |
| add v16.4s, v16.4s, v22.4s // -1+stride |
| add v17.4s, v17.4s, v23.4s |
| add v29.4s, v29.4s, v27.4s // 0, +1 |
| add v30.4s, v30.4s, v28.4s |
| add v25.4s, v25.4s, v29.4s |
| add v26.4s, v26.4s, v30.4s |
| ext v27.16b, v22.16b, v23.16b, #4 // +stride |
| ext v28.16b, v23.16b, v24.16b, #4 |
| ext v29.16b, v22.16b, v23.16b, #8 // +1+stride |
| ext v30.16b, v23.16b, v24.16b, #8 |
| .if \bpc == 8 |
| ld1 {v19.8b}, [x1], #8 // src |
| .else |
| ld1 {v19.8h}, [x1], #16 // src |
| .endif |
| add v25.4s, v25.4s, v27.4s // +stride |
| add v26.4s, v26.4s, v28.4s |
| add v16.4s, v16.4s, v29.4s // +1+stride |
| add v17.4s, v17.4s, v30.4s |
| shl v25.4s, v25.4s, #2 |
| shl v26.4s, v26.4s, #2 |
| mla v25.4s, v16.4s, v7.4s // * 3 -> b |
| mla v26.4s, v17.4s, v7.4s |
| .if \bpc == 8 |
| uxtl v19.8h, v19.8b // src |
| .endif |
| mov v0.16b, v1.16b |
| umlal v25.4s, v2.4h, v19.4h // b + a * src |
| umlal2 v26.4s, v2.8h, v19.8h |
| mov v2.16b, v3.16b |
| rshrn v25.4h, v25.4s, #9 |
| rshrn2 v25.8h, v26.4s, #9 |
| mov v4.16b, v5.16b |
| st1 {v25.8h}, [x0], #16 |
| |
| b.le 3f |
| mov v16.16b, v18.16b |
| mov v19.16b, v21.16b |
| mov v22.16b, v24.16b |
| ld1 {v1.8h}, [x9], #16 |
| ld1 {v3.8h}, [x4], #16 |
| ld1 {v5.8h}, [x10], #16 |
| ld1 {v17.4s, v18.4s}, [x7], #32 |
| ld1 {v20.4s, v21.4s}, [x3], #32 |
| ld1 {v23.4s, v24.4s}, [x8], #32 |
| b 2b |
| |
| 3: |
| subs x6, x6, #1 |
| b.le 0f |
| mov x5, x13 |
| add x0, x0, x12, lsl #1 |
| add x1, x1, x2 |
| add x3, x3, x11, lsl #2 |
| add x7, x7, x11, lsl #2 |
| add x8, x8, x11, lsl #2 |
| add x4, x4, x14, lsl #1 |
| add x9, x9, x14, lsl #1 |
| add x10, x10, x14, lsl #1 |
| b 1b |
| 0: |
| ret |
| endfunc |
| |
| // void dav1d_sgr_finish_filter2_Xbpc_neon(int16_t *tmp, |
| // const pixel *src, const ptrdiff_t stride, |
| // const int32_t *a, const int16_t *b, |
| // const int w, const int h); |
| function sgr_finish_filter2_\bpc\()bpc_neon, export=1 |
| add x7, x3, #(4*(SUM_STRIDE)) |
| sub x3, x3, #(4*(SUM_STRIDE)) |
| add x8, x4, #(2*(SUM_STRIDE)) |
| sub x4, x4, #(2*(SUM_STRIDE)) |
| mov x9, #(2*SUM_STRIDE) |
| mov x10, #FILTER_OUT_STRIDE |
| add x11, x5, #7 |
| bic x11, x11, #7 // Aligned width |
| .if \bpc == 8 |
| sub x2, x2, x11 |
| .else |
| sub x2, x2, x11, lsl #1 |
| .endif |
| sub x10, x10, x11 |
| sub x9, x9, x11 |
| sub x9, x9, #4 // We read 4 extra elements from a |
| sub x12, x9, #4 // We read 8 extra elements from b |
| mov x11, x5 |
| movi v4.8h, #5 |
| movi v5.4s, #5 |
| movi v6.8h, #6 |
| movi v7.4s, #6 |
| 1: |
| ld1 {v0.8h, v1.8h}, [x4], #32 |
| ld1 {v2.8h, v3.8h}, [x8], #32 |
| ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 |
| ld1 {v19.4s, v20.4s, v21.4s}, [x7], #48 |
| |
| 2: |
| subs x5, x5, #8 |
| ext v24.16b, v0.16b, v1.16b, #4 // +1-stride |
| ext v25.16b, v2.16b, v3.16b, #4 // +1+stride |
| ext v22.16b, v0.16b, v1.16b, #2 // -stride |
| ext v23.16b, v2.16b, v3.16b, #2 // +stride |
| add v0.8h, v0.8h, v24.8h // -1-stride, +1-stride |
| add v25.8h, v2.8h, v25.8h // -1+stride, +1+stride |
| add v2.8h, v22.8h, v23.8h // -stride, +stride |
| add v0.8h, v0.8h, v25.8h |
| |
| ext v22.16b, v16.16b, v17.16b, #4 // -stride |
| ext v23.16b, v17.16b, v18.16b, #4 |
| ext v24.16b, v19.16b, v20.16b, #4 // +stride |
| ext v25.16b, v20.16b, v21.16b, #4 |
| ext v26.16b, v16.16b, v17.16b, #8 // +1-stride |
| ext v27.16b, v17.16b, v18.16b, #8 |
| ext v28.16b, v19.16b, v20.16b, #8 // +1+stride |
| ext v29.16b, v20.16b, v21.16b, #8 |
| mul v0.8h, v0.8h, v4.8h // * 5 |
| mla v0.8h, v2.8h, v6.8h // * 6 |
| .if \bpc == 8 |
| ld1 {v31.8b}, [x1], #8 |
| .else |
| ld1 {v31.8h}, [x1], #16 |
| .endif |
| add v16.4s, v16.4s, v26.4s // -1-stride, +1-stride |
| add v17.4s, v17.4s, v27.4s |
| add v19.4s, v19.4s, v28.4s // -1+stride, +1+stride |
| add v20.4s, v20.4s, v29.4s |
| add v16.4s, v16.4s, v19.4s |
| add v17.4s, v17.4s, v20.4s |
| |
| add v22.4s, v22.4s, v24.4s // -stride, +stride |
| add v23.4s, v23.4s, v25.4s |
| // This is, surprisingly, faster than other variants where the |
| // mul+mla pairs are further apart, on Cortex A53. |
| mul v16.4s, v16.4s, v5.4s // * 5 |
| mla v16.4s, v22.4s, v7.4s // * 6 |
| mul v17.4s, v17.4s, v5.4s // * 5 |
| mla v17.4s, v23.4s, v7.4s // * 6 |
| |
| .if \bpc == 8 |
| uxtl v31.8h, v31.8b |
| .endif |
| umlal v16.4s, v0.4h, v31.4h // b + a * src |
| umlal2 v17.4s, v0.8h, v31.8h |
| mov v0.16b, v1.16b |
| rshrn v16.4h, v16.4s, #9 |
| rshrn2 v16.8h, v17.4s, #9 |
| mov v2.16b, v3.16b |
| st1 {v16.8h}, [x0], #16 |
| |
| b.le 3f |
| mov v16.16b, v18.16b |
| mov v19.16b, v21.16b |
| ld1 {v1.8h}, [x4], #16 |
| ld1 {v3.8h}, [x8], #16 |
| ld1 {v17.4s, v18.4s}, [x3], #32 |
| ld1 {v20.4s, v21.4s}, [x7], #32 |
| b 2b |
| |
| 3: |
| subs x6, x6, #1 |
| b.le 0f |
| mov x5, x11 |
| add x0, x0, x10, lsl #1 |
| add x1, x1, x2 |
| add x3, x3, x9, lsl #2 |
| add x7, x7, x9, lsl #2 |
| add x4, x4, x12, lsl #1 |
| add x8, x8, x12, lsl #1 |
| mov x13, x3 |
| mov x14, x4 |
| |
| ld1 {v0.8h, v1.8h}, [x4], #32 |
| ld1 {v16.4s, v17.4s, v18.4s}, [x3], #48 |
| |
| 4: |
| subs x5, x5, #8 |
| ext v23.16b, v0.16b, v1.16b, #4 // +1 |
| ext v22.16b, v0.16b, v1.16b, #2 // 0 |
| add v0.8h, v0.8h, v23.8h // -1, +1 |
| |
| ext v24.16b, v16.16b, v17.16b, #4 // 0 |
| ext v25.16b, v17.16b, v18.16b, #4 |
| ext v26.16b, v16.16b, v17.16b, #8 // +1 |
| ext v27.16b, v17.16b, v18.16b, #8 |
| mul v2.8h, v22.8h, v6.8h // * 6 |
| mla v2.8h, v0.8h, v4.8h // * 5 -> a |
| .if \bpc == 8 |
| ld1 {v31.8b}, [x1], #8 |
| .else |
| ld1 {v31.8h}, [x1], #16 |
| .endif |
| add v16.4s, v16.4s, v26.4s // -1, +1 |
| add v17.4s, v17.4s, v27.4s |
| .if \bpc == 8 |
| uxtl v31.8h, v31.8b |
| .endif |
| // This is, surprisingly, faster than other variants where the |
| // mul+mla pairs are further apart, on Cortex A53. |
| mul v24.4s, v24.4s, v7.4s // * 6 |
| mla v24.4s, v16.4s, v5.4s // * 5 -> b |
| mul v25.4s, v25.4s, v7.4s // * 6 |
| mla v25.4s, v17.4s, v5.4s // * 5 -> b |
| |
| umlal v24.4s, v2.4h, v31.4h // b + a * src |
| umlal2 v25.4s, v2.8h, v31.8h |
| mov v0.16b, v1.16b |
| rshrn v24.4h, v24.4s, #8 |
| rshrn2 v24.8h, v25.4s, #8 |
| mov v16.16b, v18.16b |
| st1 {v24.8h}, [x0], #16 |
| |
| b.le 5f |
| ld1 {v1.8h}, [x4], #16 |
| ld1 {v17.4s, v18.4s}, [x3], #32 |
| b 4b |
| |
| 5: |
| subs x6, x6, #1 |
| b.le 0f |
| mov x5, x11 |
| add x0, x0, x10, lsl #1 |
| add x1, x1, x2 |
| mov x3, x13 // Rewind x3/x4 to where they started |
| mov x4, x14 |
| b 1b |
| 0: |
| ret |
| endfunc |
| |
| // void dav1d_sgr_weighted1_Xbpc_neon(pixel *dst, const ptrdiff_t dst_stride, |
| // const pixel *src, const ptrdiff_t src_stride, |
| // const int16_t *t1, const int w, const int h, |
| // const int wt, const int bitdepth_max); |
| function sgr_weighted1_\bpc\()bpc_neon, export=1 |
| .if \bpc == 16 |
| ldr w8, [sp] |
| .endif |
| dup v31.8h, w7 |
| cmp x6, #2 |
| .if \bpc == 16 |
| dup v30.8h, w8 |
| .endif |
| add x9, x0, x1 |
| add x10, x2, x3 |
| add x11, x4, #2*FILTER_OUT_STRIDE |
| mov x7, #(4*FILTER_OUT_STRIDE) |
| lsl x1, x1, #1 |
| lsl x3, x3, #1 |
| add x8, x5, #7 |
| bic x8, x8, #7 // Aligned width |
| .if \bpc == 8 |
| sub x1, x1, x8 |
| sub x3, x3, x8 |
| .else |
| sub x1, x1, x8, lsl #1 |
| sub x3, x3, x8, lsl #1 |
| .endif |
| sub x7, x7, x8, lsl #1 |
| mov x8, x5 |
| b.lt 2f |
| 1: |
| .if \bpc == 8 |
| ld1 {v0.8b}, [x2], #8 |
| ld1 {v4.8b}, [x10], #8 |
| .else |
| ld1 {v0.8h}, [x2], #16 |
| ld1 {v4.8h}, [x10], #16 |
| .endif |
| ld1 {v1.8h}, [x4], #16 |
| ld1 {v5.8h}, [x11], #16 |
| subs x5, x5, #8 |
| .if \bpc == 8 |
| ushll v0.8h, v0.8b, #4 // u |
| ushll v4.8h, v4.8b, #4 // u |
| .else |
| shl v0.8h, v0.8h, #4 // u |
| shl v4.8h, v4.8h, #4 // u |
| .endif |
| sub v1.8h, v1.8h, v0.8h // t1 - u |
| sub v5.8h, v5.8h, v4.8h // t1 - u |
| ushll v2.4s, v0.4h, #7 // u << 7 |
| ushll2 v3.4s, v0.8h, #7 // u << 7 |
| ushll v6.4s, v4.4h, #7 // u << 7 |
| ushll2 v7.4s, v4.8h, #7 // u << 7 |
| smlal v2.4s, v1.4h, v31.4h // v |
| smlal2 v3.4s, v1.8h, v31.8h // v |
| smlal v6.4s, v5.4h, v31.4h // v |
| smlal2 v7.4s, v5.8h, v31.8h // v |
| .if \bpc == 8 |
| rshrn v2.4h, v2.4s, #11 |
| rshrn2 v2.8h, v3.4s, #11 |
| rshrn v6.4h, v6.4s, #11 |
| rshrn2 v6.8h, v7.4s, #11 |
| sqxtun v2.8b, v2.8h |
| sqxtun v6.8b, v6.8h |
| st1 {v2.8b}, [x0], #8 |
| st1 {v6.8b}, [x9], #8 |
| .else |
| sqrshrun v2.4h, v2.4s, #11 |
| sqrshrun2 v2.8h, v3.4s, #11 |
| sqrshrun v6.4h, v6.4s, #11 |
| sqrshrun2 v6.8h, v7.4s, #11 |
| umin v2.8h, v2.8h, v30.8h |
| umin v6.8h, v6.8h, v30.8h |
| st1 {v2.8h}, [x0], #16 |
| st1 {v6.8h}, [x9], #16 |
| .endif |
| b.gt 1b |
| |
| sub x6, x6, #2 |
| cmp x6, #1 |
| b.lt 0f |
| mov x5, x8 |
| add x0, x0, x1 |
| add x9, x9, x1 |
| add x2, x2, x3 |
| add x10, x10, x3 |
| add x4, x4, x7 |
| add x11, x11, x7 |
| b.eq 2f |
| b 1b |
| |
| 2: |
| .if \bpc == 8 |
| ld1 {v0.8b}, [x2], #8 |
| .else |
| ld1 {v0.8h}, [x2], #16 |
| .endif |
| ld1 {v1.8h}, [x4], #16 |
| subs x5, x5, #8 |
| .if \bpc == 8 |
| ushll v0.8h, v0.8b, #4 // u |
| .else |
| shl v0.8h, v0.8h, #4 // u |
| .endif |
| sub v1.8h, v1.8h, v0.8h // t1 - u |
| ushll v2.4s, v0.4h, #7 // u << 7 |
| ushll2 v3.4s, v0.8h, #7 // u << 7 |
| smlal v2.4s, v1.4h, v31.4h // v |
| smlal2 v3.4s, v1.8h, v31.8h // v |
| .if \bpc == 8 |
| rshrn v2.4h, v2.4s, #11 |
| rshrn2 v2.8h, v3.4s, #11 |
| sqxtun v2.8b, v2.8h |
| st1 {v2.8b}, [x0], #8 |
| .else |
| sqrshrun v2.4h, v2.4s, #11 |
| sqrshrun2 v2.8h, v3.4s, #11 |
| umin v2.8h, v2.8h, v30.8h |
| st1 {v2.8h}, [x0], #16 |
| .endif |
| b.gt 2b |
| 0: |
| ret |
| endfunc |
| |
| // void dav1d_sgr_weighted2_Xbpc_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *src, const ptrdiff_t src_stride, |
| // const int16_t *t1, const int16_t *t2, |
| // const int w, const int h, |
| // const int16_t wt[2], const int bitdepth_max); |
| function sgr_weighted2_\bpc\()bpc_neon, export=1 |
| .if \bpc == 8 |
| ldr x8, [sp] |
| .else |
| ldp x8, x9, [sp] |
| .endif |
| cmp x7, #2 |
| add x10, x0, x1 |
| add x11, x2, x3 |
| add x12, x4, #2*FILTER_OUT_STRIDE |
| add x13, x5, #2*FILTER_OUT_STRIDE |
| ld2r {v30.8h, v31.8h}, [x8] // wt[0], wt[1] |
| .if \bpc == 16 |
| dup v29.8h, w9 |
| .endif |
| mov x8, #4*FILTER_OUT_STRIDE |
| lsl x1, x1, #1 |
| lsl x3, x3, #1 |
| add x9, x6, #7 |
| bic x9, x9, #7 // Aligned width |
| .if \bpc == 8 |
| sub x1, x1, x9 |
| sub x3, x3, x9 |
| .else |
| sub x1, x1, x9, lsl #1 |
| sub x3, x3, x9, lsl #1 |
| .endif |
| sub x8, x8, x9, lsl #1 |
| mov x9, x6 |
| b.lt 2f |
| 1: |
| .if \bpc == 8 |
| ld1 {v0.8b}, [x2], #8 |
| ld1 {v16.8b}, [x11], #8 |
| .else |
| ld1 {v0.8h}, [x2], #16 |
| ld1 {v16.8h}, [x11], #16 |
| .endif |
| ld1 {v1.8h}, [x4], #16 |
| ld1 {v17.8h}, [x12], #16 |
| ld1 {v2.8h}, [x5], #16 |
| ld1 {v18.8h}, [x13], #16 |
| subs x6, x6, #8 |
| .if \bpc == 8 |
| ushll v0.8h, v0.8b, #4 // u |
| ushll v16.8h, v16.8b, #4 // u |
| .else |
| shl v0.8h, v0.8h, #4 // u |
| shl v16.8h, v16.8h, #4 // u |
| .endif |
| sub v1.8h, v1.8h, v0.8h // t1 - u |
| sub v2.8h, v2.8h, v0.8h // t2 - u |
| sub v17.8h, v17.8h, v16.8h // t1 - u |
| sub v18.8h, v18.8h, v16.8h // t2 - u |
| ushll v3.4s, v0.4h, #7 // u << 7 |
| ushll2 v4.4s, v0.8h, #7 // u << 7 |
| ushll v19.4s, v16.4h, #7 // u << 7 |
| ushll2 v20.4s, v16.8h, #7 // u << 7 |
| smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) |
| smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) |
| smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) |
| smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) |
| smlal v19.4s, v17.4h, v30.4h // wt[0] * (t1 - u) |
| smlal v19.4s, v18.4h, v31.4h // wt[1] * (t2 - u) |
| smlal2 v20.4s, v17.8h, v30.8h // wt[0] * (t1 - u) |
| smlal2 v20.4s, v18.8h, v31.8h // wt[1] * (t2 - u) |
| .if \bpc == 8 |
| rshrn v3.4h, v3.4s, #11 |
| rshrn2 v3.8h, v4.4s, #11 |
| rshrn v19.4h, v19.4s, #11 |
| rshrn2 v19.8h, v20.4s, #11 |
| sqxtun v3.8b, v3.8h |
| sqxtun v19.8b, v19.8h |
| st1 {v3.8b}, [x0], #8 |
| st1 {v19.8b}, [x10], #8 |
| .else |
| sqrshrun v3.4h, v3.4s, #11 |
| sqrshrun2 v3.8h, v4.4s, #11 |
| sqrshrun v19.4h, v19.4s, #11 |
| sqrshrun2 v19.8h, v20.4s, #11 |
| umin v3.8h, v3.8h, v29.8h |
| umin v19.8h, v19.8h, v29.8h |
| st1 {v3.8h}, [x0], #16 |
| st1 {v19.8h}, [x10], #16 |
| .endif |
| b.gt 1b |
| |
| subs x7, x7, #2 |
| cmp x7, #1 |
| b.lt 0f |
| mov x6, x9 |
| add x0, x0, x1 |
| add x10, x10, x1 |
| add x2, x2, x3 |
| add x11, x11, x3 |
| add x4, x4, x8 |
| add x12, x12, x8 |
| add x5, x5, x8 |
| add x13, x13, x8 |
| b.eq 2f |
| b 1b |
| |
| 2: |
| .if \bpc == 8 |
| ld1 {v0.8b}, [x2], #8 |
| .else |
| ld1 {v0.8h}, [x2], #16 |
| .endif |
| ld1 {v1.8h}, [x4], #16 |
| ld1 {v2.8h}, [x5], #16 |
| subs x6, x6, #8 |
| .if \bpc == 8 |
| ushll v0.8h, v0.8b, #4 // u |
| .else |
| shl v0.8h, v0.8h, #4 // u |
| .endif |
| sub v1.8h, v1.8h, v0.8h // t1 - u |
| sub v2.8h, v2.8h, v0.8h // t2 - u |
| ushll v3.4s, v0.4h, #7 // u << 7 |
| ushll2 v4.4s, v0.8h, #7 // u << 7 |
| smlal v3.4s, v1.4h, v30.4h // wt[0] * (t1 - u) |
| smlal v3.4s, v2.4h, v31.4h // wt[1] * (t2 - u) |
| smlal2 v4.4s, v1.8h, v30.8h // wt[0] * (t1 - u) |
| smlal2 v4.4s, v2.8h, v31.8h // wt[1] * (t2 - u) |
| .if \bpc == 8 |
| rshrn v3.4h, v3.4s, #11 |
| rshrn2 v3.8h, v4.4s, #11 |
| sqxtun v3.8b, v3.8h |
| st1 {v3.8b}, [x0], #8 |
| .else |
| sqrshrun v3.4h, v3.4s, #11 |
| sqrshrun2 v3.8h, v4.4s, #11 |
| umin v3.8h, v3.8h, v29.8h |
| st1 {v3.8h}, [x0], #16 |
| .endif |
| b.gt 1b |
| 0: |
| ret |
| endfunc |
| .endm |
