| /* |
| * Copyright © 2018, VideoLAN and dav1d authors |
| * Copyright © 2019, Martin Storsjo |
| * Copyright © 2019, B Krishnan Iyer |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * |
| * 1. Redistributions of source code must retain the above copyright notice, this |
| * list of conditions and the following disclaimer. |
| * |
| * 2. Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "src/arm/asm.S" |
| #include "util.S" |
| |
| // void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_dc_128_neon, export=1 |
| push {r4, lr} |
| ldr r4, [sp, #8] |
| clz r3, r3 |
| adr r2, L(ipred_dc_128_tbl) |
| sub r3, r3, #25 |
| ldr r3, [r2, r3, lsl #2] |
| mov lr, #128 |
| vdup.8 q0, lr |
| add r2, r2, r3 |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r2 |
| |
| .align 2 |
| L(ipred_dc_128_tbl): |
| .word 640f - L(ipred_dc_128_tbl) + CONFIG_THUMB |
| .word 320f - L(ipred_dc_128_tbl) + CONFIG_THUMB |
| .word 16f - L(ipred_dc_128_tbl) + CONFIG_THUMB |
| .word 8f - L(ipred_dc_128_tbl) + CONFIG_THUMB |
| .word 4f - L(ipred_dc_128_tbl) + CONFIG_THUMB |
| 4: |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| subs r4, r4, #4 |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt 4b |
| pop {r4, pc} |
| 8: |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt 8b |
| pop {r4, pc} |
| 16: |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| bgt 16b |
| pop {r4, pc} |
| 320: |
| vdup.8 q1, lr |
| 32: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| bgt 32b |
| pop {r4, pc} |
| 640: |
| vdup.8 q1, lr |
| vdup.8 q2, lr |
| vdup.8 q3, lr |
| sub r1, r1, #32 |
| 64: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| bgt 64b |
| pop {r4, pc} |
| endfunc |
| |
| // void ipred_v_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_v_neon, export=1 |
| push {r4, lr} |
| ldr lr, [sp, #8] |
| clz r3, r3 |
| adr r4, L(ipred_v_tbl) |
| sub r3, r3, #25 |
| ldr r3, [r4, r3, lsl #2] |
| add r2, r2, #1 |
| add r4, r4, r3 |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r4 |
| |
| .align 2 |
| L(ipred_v_tbl): |
| .word 640f - L(ipred_v_tbl) + CONFIG_THUMB |
| .word 320f - L(ipred_v_tbl) + CONFIG_THUMB |
| .word 160f - L(ipred_v_tbl) + CONFIG_THUMB |
| .word 80f - L(ipred_v_tbl) + CONFIG_THUMB |
| .word 40f - L(ipred_v_tbl) + CONFIG_THUMB |
| 40: |
| vld1.32 {d0[0]}, [r2] |
| 4: |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| subs lr, lr, #4 |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt 4b |
| pop {r4, pc} |
| 80: |
| vld1.8 {d0}, [r2] |
| 8: |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| subs lr, lr, #4 |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt 8b |
| pop {r4, pc} |
| 160: |
| vld1.8 {q0}, [r2] |
| 16: |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| subs lr, lr, #4 |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| bgt 16b |
| pop {r4, pc} |
| 320: |
| vld1.8 {q0, q1}, [r2] |
| 32: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| subs lr, lr, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| bgt 32b |
| pop {r4, pc} |
| 640: |
| vld1.8 {q0, q1}, [r2]! |
| sub r1, r1, #32 |
| vld1.8 {q2, q3}, [r2] |
| 64: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| subs lr, lr, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| bgt 64b |
| pop {r4, pc} |
| endfunc |
| |
| // void ipred_h_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_h_neon, export=1 |
| push {r4-r5, lr} |
| ldr r4, [sp, #12] |
| clz r3, r3 |
| adr r5, L(ipred_h_tbl) |
| sub r3, r3, #25 |
| ldr r3, [r5, r3, lsl #2] |
| sub r2, r2, #4 |
| mov lr, #-4 |
| add r5, r5, r3 |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r5 |
| |
| .align 2 |
| L(ipred_h_tbl): |
| .word 640f - L(ipred_h_tbl) + CONFIG_THUMB |
| .word 320f - L(ipred_h_tbl) + CONFIG_THUMB |
| .word 160f - L(ipred_h_tbl) + CONFIG_THUMB |
| .word 8f - L(ipred_h_tbl) + CONFIG_THUMB |
| .word 4f - L(ipred_h_tbl) + CONFIG_THUMB |
| 4: |
| vld4.8 {d0[], d1[], d2[], d3[]}, [r2], lr |
| vst1.32 {d3[0]}, [r0, :32], r1 |
| vst1.32 {d2[0]}, [r12, :32], r1 |
| subs r4, r4, #4 |
| vst1.32 {d1[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt 4b |
| pop {r4-r5, pc} |
| 8: |
| vld4.8 {d0[], d1[], d2[], d3[]}, [r2], lr |
| vst1.8 {d3}, [r0, :64], r1 |
| vst1.8 {d2}, [r12, :64], r1 |
| subs r4, r4, #4 |
| vst1.8 {d1}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt 8b |
| pop {r4-r5, pc} |
| 160: |
| add r2, r2, #3 |
| mov lr, #-1 |
| 16: |
| vld1.8 {d0[], d1[]}, [r2], lr |
| subs r4, r4, #4 |
| vld1.8 {d2[], d3[]}, [r2], lr |
| vst1.8 {q0}, [r0, :128], r1 |
| vld1.8 {d4[], d5[]}, [r2], lr |
| vst1.8 {q1}, [r12, :128], r1 |
| vld1.8 {d6[], d7[]}, [r2], lr |
| vst1.8 {q2}, [r0, :128], r1 |
| vst1.8 {q3}, [r12, :128], r1 |
| bgt 16b |
| pop {r4-r5, pc} |
| 320: |
| add r2, r2, #3 |
| mov lr, #-1 |
| sub r1, r1, #16 |
| 32: |
| vld1.8 {d0[], d1[]}, [r2], lr |
| subs r4, r4, #4 |
| vld1.8 {d2[], d3[]}, [r2], lr |
| vst1.8 {q0}, [r0, :128]! |
| vld1.8 {d4[], d5[]}, [r2], lr |
| vst1.8 {q1}, [r12, :128]! |
| vld1.8 {d6[], d7[]}, [r2], lr |
| vst1.8 {q0}, [r0, :128], r1 |
| vst1.8 {q1}, [r12, :128], r1 |
| vst1.8 {q2}, [r0, :128]! |
| vst1.8 {q3}, [r12, :128]! |
| vst1.8 {q2}, [r0, :128], r1 |
| vst1.8 {q3}, [r12, :128], r1 |
| bgt 32b |
| pop {r4-r5, pc} |
| 640: |
| add r2, r2, #3 |
| mov lr, #-1 |
| sub r1, r1, #48 |
| 64: |
| vld1.8 {d0[], d1[]}, [r2], lr |
| subs r4, r4, #4 |
| vld1.8 {d2[], d3[]}, [r2], lr |
| vst1.8 {q0}, [r0, :128]! |
| vld1.8 {d4[], d5[]}, [r2], lr |
| vst1.8 {q1}, [r12, :128]! |
| vld1.8 {d6[], d7[]}, [r2], lr |
| vst1.8 {q0}, [r0, :128]! |
| vst1.8 {q1}, [r12, :128]! |
| vst1.8 {q0}, [r0, :128]! |
| vst1.8 {q1}, [r12, :128]! |
| vst1.8 {q0}, [r0, :128], r1 |
| vst1.8 {q1}, [r12, :128], r1 |
| vst1.8 {q2}, [r0, :128]! |
| vst1.8 {q3}, [r12, :128]! |
| vst1.8 {q2}, [r0, :128]! |
| vst1.8 {q3}, [r12, :128]! |
| vst1.8 {q2}, [r0, :128]! |
| vst1.8 {q3}, [r12, :128]! |
| vst1.8 {q2}, [r0, :128], r1 |
| vst1.8 {q3}, [r12, :128], r1 |
| bgt 64b |
| pop {r4-r5, pc} |
| endfunc |
| |
| // void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_dc_top_neon, export=1 |
| push {r4-r5, lr} |
| ldr r4, [sp, #12] |
| clz r3, r3 |
| adr r5, L(ipred_dc_top_tbl) |
| sub r3, r3, #25 |
| ldr r3, [r5, r3, lsl #2] |
| add r2, r2, #1 |
| add r5, r5, r3 |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r5 |
| |
| .align 2 |
| L(ipred_dc_top_tbl): |
| .word 640f - L(ipred_dc_top_tbl) + CONFIG_THUMB |
| .word 320f - L(ipred_dc_top_tbl) + CONFIG_THUMB |
| .word 160f - L(ipred_dc_top_tbl) + CONFIG_THUMB |
| .word 80f - L(ipred_dc_top_tbl) + CONFIG_THUMB |
| .word 40f - L(ipred_dc_top_tbl) + CONFIG_THUMB |
| 40: |
| vld1.32 {d0[]}, [r2] |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #2 |
| vdup.8 d0, d0[0] |
| 4: |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| subs r4, r4, #4 |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt 4b |
| pop {r4-r5, pc} |
| 80: |
| vld1.8 {d0}, [r2] |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #3 |
| vdup.8 d0, d0[0] |
| 8: |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt 8b |
| pop {r4-r5, pc} |
| 160: |
| vld1.8 {d0, d1}, [r2] |
| vaddl.u8 q0, d0, d1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #4 |
| vdup.8 q0, d0[0] |
| 16: |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| bgt 16b |
| pop {r4-r5, pc} |
| 320: |
| vld1.8 {d0, d1, d2, d3}, [r2] |
| vaddl.u8 q0, d0, d1 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d4, q0, #5 |
| vdup.8 q0, d4[0] |
| vdup.8 q1, d4[0] |
| 32: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| bgt 32b |
| pop {r4-r5, pc} |
| 640: |
| vld1.8 {d0, d1, d2, d3}, [r2]! |
| vaddl.u8 q0, d0, d1 |
| vld1.8 {d4, d5, d6, d7}, [r2] |
| vaddl.u8 q1, d2, d3 |
| vaddl.u8 q2, d4, d5 |
| vaddl.u8 q3, d6, d7 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 q1, q2, q3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d18, q0, #6 |
| vdup.8 q0, d18[0] |
| vdup.8 q1, d18[0] |
| vdup.8 q2, d18[0] |
| vdup.8 q3, d18[0] |
| sub r1, r1, #32 |
| 64: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| bgt 64b |
| pop {r4-r5, pc} |
| endfunc |
| |
| // void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_dc_left_neon, export=1 |
| push {r4-r5, lr} |
| ldr r4, [sp, #12] |
| sub r2, r2, r4 |
| clz r3, r3 |
| clz lr, r4 |
| sub lr, lr, #25 |
| adr r5, L(ipred_dc_left_tbl) |
| sub r3, r3, #20 |
| ldr r3, [r5, r3, lsl #2] |
| ldr lr, [r5, lr, lsl #2] |
| add r3, r5, r3 |
| add r5, r5, lr |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r5 |
| |
| .align 2 |
| L(ipred_dc_left_tbl): |
| .word L(ipred_dc_left_h64) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_h32) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_h16) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_h8) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_h4) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_w64) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_w32) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_w8) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_left_w4) - L(ipred_dc_left_tbl) + CONFIG_THUMB |
| |
| L(ipred_dc_left_h4): |
| vld1.32 {d0[]}, [r2] |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #2 |
| vdup.8 q0, d0[0] |
| bx r3 |
| L(ipred_dc_left_w4): |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| subs r4, r4, #4 |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt L(ipred_dc_left_w4) |
| pop {r4-r5, pc} |
| L(ipred_dc_left_h8): |
| vld1.8 {d0}, [r2] |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #3 |
| vdup.8 q0, d0[0] |
| bx r3 |
| L(ipred_dc_left_w8): |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt L(ipred_dc_left_w8) |
| pop {r4-r5, pc} |
| L(ipred_dc_left_h16): |
| vld1.8 {d0, d1}, [r2] |
| vaddl.u8 q0, d0, d1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #4 |
| vdup.8 q0, d0[0] |
| bx r3 |
| L(ipred_dc_left_w16): |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| bgt L(ipred_dc_left_w16) |
| pop {r4-r5, pc} |
| L(ipred_dc_left_h32): |
| vld1.8 {d0, d1, d2, d3}, [r2] |
| vaddl.u8 q0, d0, d1 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #5 |
| vdup.8 q0, d0[0] |
| bx r3 |
| L(ipred_dc_left_w32): |
| vmov.8 q1, q0 |
| 1: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| bgt 1b |
| pop {r4-r5, pc} |
| L(ipred_dc_left_h64): |
| vld1.8 {d0, d1, d2, d3}, [r2]! |
| vld1.8 {d4, d5, d6, d7}, [r2] |
| vaddl.u8 q0, d0, d1 |
| vaddl.u8 q1, d2, d3 |
| vaddl.u8 q2, d4, d5 |
| vaddl.u8 q3, d6, d7 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 q1, q2, q3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| vrshrn.u16 d0, q0, #6 |
| vdup.8 q0, d0[0] |
| bx r3 |
| L(ipred_dc_left_w64): |
| sub r1, r1, #32 |
| vmov.8 q1, q0 |
| vmov.8 q2, q0 |
| vmov.8 q3, q0 |
| 1: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| bgt 1b |
| pop {r4-r5, pc} |
| endfunc |
| |
| // void ipred_dc_neon(pixel *dst, const ptrdiff_t stride, |
| // const pixel *const topleft, |
| // const int width, const int height, const int a, |
| // const int max_width, const int max_height); |
| function ipred_dc_neon, export=1 |
| push {r4-r6, lr} |
| ldr r4, [sp, #16] |
| sub r2, r2, r4 |
| add lr, r3, r4 // width + height |
| clz r3, r3 |
| clz r12, r4 |
| vdup.16 q15, lr // width + height |
| mov r6, #0 |
| adr r5, L(ipred_dc_tbl) |
| rbit lr, lr // rbit(width + height) |
| sub r3, r3, #20 // 25 leading bits, minus table offset 5 |
| sub r12, r12, #25 |
| clz lr, lr // ctz(width + height) |
| ldr r3, [r5, r3, lsl #2] |
| ldr r12, [r5, r12, lsl #2] |
| neg lr, lr // -ctz(width + height) |
| add r3, r5, r3 |
| add r5, r5, r12 |
| vshr.u16 q15, q15, #1 // (width + height) >> 1 |
| vdup.16 q14, lr // -ctz(width + height) |
| add r12, r0, r1 |
| lsl r1, r1, #1 |
| bx r5 |
| |
| .align 2 |
| L(ipred_dc_tbl): |
| .word L(ipred_dc_h64) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_h32) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_h16) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_h8) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_h4) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_w64) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_w32) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_w8) - L(ipred_dc_tbl) + CONFIG_THUMB |
| .word L(ipred_dc_w4) - L(ipred_dc_tbl) + CONFIG_THUMB |
| |
| L(ipred_dc_h4): |
| vld1.32 {d0[0]}, [r2]! |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| bx r3 |
| L(ipred_dc_w4): |
| add r2, r2, #1 |
| vld1.32 {d1[0]}, [r2] |
| vmov.32 d1[1], r6 |
| vadd.s16 d0, d0, d30 |
| vpaddl.u8 d1, d1 |
| vpadd.u16 d1, d1 |
| vpadd.u16 d1, d1 |
| cmp r4, #4 |
| vadd.s16 d0, d0, d1 |
| vshl.u16 d0, d0, d28 |
| beq 1f // h = 8/16 |
| movw lr, #(0x3334/2) |
| movw r5, #(0x5556/2) |
| cmp r4, #16 |
| it ne |
| movne lr, r5 |
| vdup.16 d30, lr |
| vqdmulh.s16 d0, d0, d30 |
| 1: |
| vdup.8 d0, d0[0] |
| 2: |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| subs r4, r4, #4 |
| vst1.32 {d0[0]}, [r0, :32], r1 |
| vst1.32 {d0[0]}, [r12, :32], r1 |
| bgt 2b |
| pop {r4-r6, pc} |
| |
| L(ipred_dc_h8): |
| vld1.8 {d0}, [r2]! |
| vpaddl.u8 d0, d0 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| bx r3 |
| L(ipred_dc_w8): |
| add r2, r2, #1 |
| vld1.8 {d2}, [r2] |
| vadd.s16 d0, d0, d30 |
| vpaddl.u8 d2, d2 |
| vpadd.u16 d2, d2 |
| vpadd.u16 d2, d2 |
| cmp r4, #8 |
| vadd.s16 d0, d0, d2 |
| vshl.u16 d0, d0, d28 |
| beq 1f // h = 4/16/32 |
| cmp r4, #32 |
| movw lr, #(0x3334/2) |
| movw r5, #(0x5556/2) |
| it ne |
| movne lr, r5 |
| vdup.16 q12, lr |
| vqdmulh.s16 d0, d0, d24 |
| 1: |
| vdup.8 d0, d0[0] |
| 2: |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0}, [r0, :64], r1 |
| vst1.8 {d0}, [r12, :64], r1 |
| bgt 2b |
| pop {r4-r6, pc} |
| |
| L(ipred_dc_h16): |
| vld1.8 {d0, d1}, [r2]! |
| vaddl.u8 q0, d0, d1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| bx r3 |
| L(ipred_dc_w16): |
| add r2, r2, #1 |
| vld1.8 {d2, d3}, [r2] |
| vadd.s16 d0, d0, d30 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 d2, d2, d3 |
| vpadd.u16 d2, d2 |
| vpadd.u16 d2, d2 |
| cmp r4, #16 |
| vadd.s16 d0, d0, d2 |
| vshl.u16 d0, d0, d28 |
| beq 1f // h = 4/8/32/64 |
| tst r4, #(32+16+8) // 16 added to make a consecutive bitmask |
| movw lr, #(0x3334/2) |
| movw r5, #(0x5556/2) |
| it ne |
| movne lr, r5 |
| vdup.16 q12, lr |
| vqdmulh.s16 d0, d0, d24 |
| 1: |
| vdup.8 q0, d0[0] |
| 2: |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1}, [r0, :128], r1 |
| vst1.8 {d0, d1}, [r12, :128], r1 |
| bgt 2b |
| pop {r4-r6, pc} |
| |
| L(ipred_dc_h32): |
| vld1.8 {d0, d1, d2, d3}, [r2]! |
| vaddl.u8 q0, d0, d1 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| bx r3 |
| L(ipred_dc_w32): |
| add r2, r2, #1 |
| vld1.8 {d2, d3, d4, d5}, [r2] |
| vadd.s16 d0, d0, d30 |
| vaddl.u8 q2, d4, d5 |
| vadd.u16 d4, d4, d5 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 d2, d2, d3 |
| vpadd.u16 d4, d4 |
| vpadd.u16 d2, d2 |
| vpadd.u16 d4, d4 |
| vpadd.u16 d2, d2 |
| cmp r4, #32 |
| vadd.s16 d0, d0, d4 |
| vadd.s16 d0, d0, d2 |
| vshl.u16 d4, d0, d28 |
| beq 1f // h = 8/16/64 |
| cmp r4, #8 |
| movw lr, #(0x3334/2) |
| movw r5, #(0x5556/2) |
| it ne |
| movne lr, r5 |
| vdup.16 q12, lr |
| vqdmulh.s16 d4, d4, d24 |
| 1: |
| vdup.8 q0, d4[0] |
| vdup.8 q1, d4[0] |
| 2: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 |
| vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 |
| bgt 2b |
| pop {r4-r6, pc} |
| |
| L(ipred_dc_h64): |
| vld1.8 {d0, d1, d2, d3}, [r2]! |
| vaddl.u8 q0, d0, d1 |
| vld1.8 {d4, d5, d6, d7}, [r2]! |
| vaddl.u8 q1, d2, d3 |
| vaddl.u8 q2, d4, d5 |
| vaddl.u8 q3, d6, d7 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 q1, q2, q3 |
| vadd.u16 q0, q0, q1 |
| vadd.u16 d0, d0, d1 |
| vpadd.u16 d0, d0 |
| vpadd.u16 d0, d0 |
| bx r3 |
| L(ipred_dc_w64): |
| vmov.8 q1, q0 |
| vmov.8 q2, q0 |
| vmov.8 q3, q0 |
| 2: |
| add r2, r2, #1 |
| vld1.8 {d2, d3, d4, d5}, [r2]! |
| vadd.s16 d0, d0, d30 |
| vaddl.u8 q2, d4, d5 |
| vaddl.u8 q1, d2, d3 |
| vadd.u16 d4, d4, d5 |
| vadd.u16 d2, d2, d3 |
| vld1.8 {d16, d17, d18, d19}, [r2] |
| vpadd.u16 d4, d4 |
| vpadd.u16 d2, d2 |
| vpadd.u16 d4, d4 |
| vpadd.u16 d2, d2 |
| vaddl.u8 q8, d16, d17 |
| vaddl.u8 q9, d18, d19 |
| vadd.u16 d16, d16, d17 |
| vadd.u16 d18, d18, d19 |
| vpadd.u16 d16, d16 |
| vpadd.u16 d18, d18 |
| vpadd.u16 d16, d16 |
| vpadd.u16 d18, d18 |
| vadd.u16 d2, d2, d4 |
| vadd.u16 d3, d16, d18 |
| cmp r4, #64 |
| vadd.s16 d0, d0, d2 |
| vadd.s16 d0, d0, d3 |
| vshl.u16 d18, d0, d28 |
| beq 1f // h = 16/32 |
| movw lr, #(0x5556/2) |
| movt lr, #(0x3334/2) |
| mov r5, r4 |
| and r5, r5, #31 |
| lsr lr, lr, r5 |
| vdup.16 d30, lr |
| vqdmulh.s16 d18, d18, d30 |
| 1: |
| sub r1, r1, #32 |
| vdup.8 q0, d18[0] |
| vdup.8 q1, d18[0] |
| vdup.8 q2, d18[0] |
| vdup.8 q3, d18[0] |
| 2: |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| subs r4, r4, #4 |
| vst1.8 {d0, d1, d2, d3}, [r0, :128]! |
| vst1.8 {d0, d1, d2, d3}, [r12, :128]! |
| vst1.8 {d4, d5, d6, d7}, [r0, :128], r1 |
| vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 |
| bgt 2b |
| pop {r4-r6, pc} |
| endfunc |
| |
