| /* |
| * Copyright © 2021, VideoLAN and dav1d authors |
| * Copyright © 2021, Martin Storsjo |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are met: |
| * |
| * 1. Redistributions of source code must retain the above copyright notice, this |
| * list of conditions and the following disclaimer. |
| * |
| * 2. Redistributions in binary form must reproduce the above copyright notice, |
| * this list of conditions and the following disclaimer in the documentation |
| * and/or other materials provided with the distribution. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
| * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
| * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR |
| * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
| * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
| * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| #include "src/arm/asm.S" |
| #include "util.S" |
| #include "src/arm/asm-offsets.h" |
| |
| #define GRAIN_WIDTH 82 |
| #define GRAIN_HEIGHT 73 |
| |
| #define SUB_GRAIN_WIDTH 44 |
| #define SUB_GRAIN_HEIGHT 38 |
| |
| .macro increment_seed steps, shift=1 |
| lsr r11, r2, #3 |
| lsr r12, r2, #12 |
| lsr lr, r2, #1 |
| eor r11, r2, r11 // (r >> 0) ^ (r >> 3) |
| eor r12, r12, lr // (r >> 12) ^ (r >> 1) |
| eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) |
| .if \shift |
| lsr r2, r2, #\steps |
| .endif |
| and r11, r11, #((1 << \steps) - 1) // bit |
| .if \shift |
| orr r2, r2, r11, lsl #(16 - \steps) // *state |
| .else |
| orr r2, r2, r11, lsl #16 // *state |
| .endif |
| .endm |
| |
| .macro read_rand dest, bits, age |
| ubfx \dest, r2, #16 - \bits - \age, #\bits |
| .endm |
| |
| .macro read_shift_rand dest, bits |
| ubfx \dest, r2, #17 - \bits, #\bits |
| lsr r2, r2, #1 |
| .endm |
| |
| // special calling convention: |
| // r2 holds seed |
| // r3 holds dav1d_gaussian_sequence |
| // clobbers r11-r12 |
| // returns in d0-d1 |
| function get_gaussian_neon |
| push {r5-r6,lr} |
| increment_seed 4 |
| read_rand r5, 11, 3 |
| read_rand r6, 11, 2 |
| add r5, r3, r5, lsl #1 |
| add r6, r3, r6, lsl #1 |
| vld1.16 {d0[0]}, [r5] |
| read_rand r5, 11, 1 |
| vld1.16 {d0[1]}, [r6] |
| add r5, r3, r5, lsl #1 |
| read_rand r6, 11, 0 |
| increment_seed 4 |
| add r6, r3, r6, lsl #1 |
| vld1.16 {d0[2]}, [r5] |
| read_rand r5, 11, 3 |
| vld1.16 {d0[3]}, [r6] |
| add r5, r3, r5, lsl #1 |
| read_rand r6, 11, 2 |
| vld1.16 {d1[0]}, [r5] |
| add r6, r3, r6, lsl #1 |
| read_rand r5, 11, 1 |
| vld1.16 {d1[1]}, [r6] |
| read_rand r6, 11, 0 |
| add r5, r3, r5, lsl #1 |
| add r6, r3, r6, lsl #1 |
| vld1.16 {d1[2]}, [r5] |
| vld1.16 {d1[3]}, [r6] |
| pop {r5-r6,pc} |
| endfunc |
| |
| function get_grain_2_neon |
| push {r11,lr} |
| increment_seed 2 |
| read_rand r11, 11, 1 |
| read_rand r12, 11, 0 |
| add r11, r3, r11, lsl #1 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d0[0]}, [r11] |
| vld1.16 {d0[1]}, [r12] |
| vrshl.s16 d0, d0, d30 |
| pop {r11,pc} |
| endfunc |
| |
| .macro get_grain_2 dst |
| bl get_grain_2_neon |
| .ifnc \dst, d0 |
| vmov \dst, d0 |
| .endif |
| .endm |
| |
| function get_grain_4_neon |
| push {r11,lr} |
| increment_seed 4 |
| read_rand r11, 11, 3 |
| read_rand r12, 11, 2 |
| add r11, r3, r11, lsl #1 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d0[0]}, [r11] |
| read_rand r11, 11, 1 |
| vld1.16 {d0[1]}, [r12] |
| read_rand r12, 11, 0 |
| add r11, r3, r11, lsl #1 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d0[2]}, [r11] |
| vld1.16 {d0[3]}, [r12] |
| vrshl.s16 d0, d0, d30 |
| pop {r11,pc} |
| endfunc |
| |
| .macro get_grain_4 dst |
| bl get_grain_4_neon |
| .ifnc \dst, d0 |
| vmov \dst, d0 |
| .endif |
| .endm |
| |
| // r1 holds the number of entries to produce |
| // r6, r8 and r10 hold the previous output entries |
| // q0 holds the vector of produced entries |
| // q1 holds the input vector of sums from above |
| .macro output_lag n |
| function output_lag\n\()_neon |
| push {r0, lr} |
| .if \n == 1 |
| mvn lr, r5 // grain_min = ~grain_max |
| .else |
| mov r0, #1 |
| mov lr, #1 |
| sub r7, r7, #1 |
| sub r9, r9, #1 |
| lsl r0, r0, r7 |
| lsl lr, lr, r9 |
| add r7, r7, #1 |
| add r9, r9, #1 |
| .endif |
| 1: |
| read_shift_rand r12, 11 |
| vmov.32 r11, d2[0] |
| lsl r12, r12, #1 |
| vext.8 q0, q0, q0, #2 |
| ldrsh r12, [r3, r12] |
| .if \n == 1 |
| mla r11, r6, r4, r11 // sum (above) + *coeff * prev output |
| add r6, r11, r8 // 1 << (ar_coeff_shift - 1) |
| add r12, r12, r10 |
| asr r6, r6, r7 // >> ar_coeff_shift |
| asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) |
| add r6, r6, r12 |
| cmp r6, r5 |
| .elseif \n == 2 |
| mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 |
| mla r11, r6, r10, r11 // += *coeff * prev output 2 |
| mov r8, r6 |
| add r6, r11, r0 // 1 << (ar_coeff_shift - 1) |
| add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) |
| asr r6, r6, r7 // >> ar_coeff_shift |
| asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) |
| add r6, r6, r12 |
| push {lr} |
| cmp r6, r5 |
| mvn lr, r5 // grain_min = ~grain_max |
| .else |
| push {r1-r3} |
| sbfx r1, r4, #0, #8 |
| sbfx r2, r4, #8, #8 |
| sbfx r3, r4, #16, #8 |
| mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 |
| mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 |
| mla r11, r6, r3, r11 // += *coeff * prev output 3 |
| pop {r1-r3} |
| mov r10, r8 |
| mov r8, r6 |
| |
| add r6, r11, r0 // 1 << (ar_coeff_shift - 1) |
| add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) |
| asr r6, r6, r7 // >> ar_coeff_shift |
| asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) |
| add r6, r6, r12 |
| push {lr} |
| cmp r6, r5 |
| mvn lr, r5 // grain_min = ~grain_max |
| .endif |
| it gt |
| movgt r6, r5 |
| cmp r6, lr |
| it lt |
| movlt r6, lr |
| .if \n >= 2 |
| pop {lr} |
| .endif |
| subs r1, r1, #1 |
| vext.8 q1, q1, q1, #4 |
| vmov.16 d1[3], r6 |
| bgt 1b |
| pop {r0, pc} |
| endfunc |
| .endm |
| |
| output_lag 1 |
| output_lag 2 |
| output_lag 3 |
| |
| |
| function sum_lag1_above_neon |
| sub r12, r0, #1*GRAIN_WIDTH*2 - 16 |
| vld1.16 {q10}, [r12] // load top right |
| |
| vext.8 q0, q8, q9, #14 // top left, top mid |
| vext.8 q1, q9, q10, #2 // top left, top mid |
| |
| vmull.s16 q2, d18, d28 |
| vmlal.s16 q2, d0, d27 |
| vmlal.s16 q2, d2, d29 |
| vmull.s16 q3, d19, d28 |
| vmlal.s16 q3, d1, d27 |
| vmlal.s16 q3, d3, d29 |
| |
| vmov q8, q9 |
| vmov q9, q10 |
| |
| bx lr |
| endfunc |
| |
| .macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff |
| .ifc \lag\()_\edge, lag3_left |
| bl sum_lag3_left_above_neon |
| .else |
| bl sum_\lag\()_above_neon |
| .endif |
| .ifc \type, uv_420 |
| vpush {q6-q7} |
| add r12, r11, #GRAIN_WIDTH*2 |
| vld1.16 {q0, q1}, [r11]! |
| vld1.16 {q6, q7}, [r12]! |
| vpadd.i16 d0, d0, d1 |
| vpadd.i16 d1, d2, d3 |
| vpadd.i16 d12, d12, d13 |
| vpadd.i16 d13, d14, d15 |
| vadd.i16 q0, q0, q6 |
| vpop {q6-q7} |
| vrshr.s16 q0, q0, #2 |
| .endif |
| .ifc \type, uv_422 |
| vld1.16 {q0, q1}, [r11]! |
| vpadd.i16 d0, d0, d1 |
| vpadd.i16 d1, d2, d3 |
| vrshr.s16 q0, q0, #1 |
| .endif |
| .ifc \type, uv_444 |
| vld1.16 {q0}, [r11]! |
| .endif |
| .if \uv_layout |
| .ifnb \uv_coeff |
| vdup.8 d13, \uv_coeff |
| vmovl.s8 q6, d13 |
| .endif |
| vmlal.s16 q2, d0, d13 |
| vmlal.s16 q3, d1, d13 |
| .endif |
| .if \uv_layout && \elems == 8 |
| b sum_\lag\()_y_\edge\()_start |
| .elseif \uv_layout == 444 && \elems == 7 |
| b sum_\lag\()_y_\edge\()_start |
| .elseif \uv_layout == 422 && \elems == 1 |
| b sum_\lag\()_uv_420_\edge\()_start |
| .else |
| sum_\lag\()_\type\()_\edge\()_start: |
| push {r11} |
| .if \elems > 4 |
| .ifc \edge, left |
| increment_seed 4 |
| read_rand r11, 11, 3 |
| read_rand r12, 11, 2 |
| add r11, r3, r11, lsl #1 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d1[1]}, [r11] |
| read_rand r11, 11, 1 |
| vld1.16 {d1[2]}, [r12] |
| add r11, r3, r11, lsl #1 |
| vld1.16 {d1[3]}, [r11] |
| lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 |
| vrshl.s16 d1, d1, d30 |
| vext.8 q2, q2, q2, #12 |
| .ifc \lag, lag3 |
| vmov.s16 r10, d1[1] |
| .endif |
| .ifnc \lag, lag1 |
| vmov.s16 r8, d1[2] |
| .endif |
| vmov.s16 r6, d1[3] |
| |
| vmov q1, q2 |
| mov r1, #1 |
| bl output_\lag\()_neon |
| .else |
| increment_seed 4, shift=0 |
| vmov q1, q2 |
| mov r1, #4 |
| bl output_\lag\()_neon |
| .endif |
| |
| increment_seed 4, shift=0 |
| vmov q1, q3 |
| .ifc \edge, right |
| mov r1, #3 |
| bl output_\lag\()_neon |
| read_shift_rand r12, 11 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d2[0]}, [r12] |
| vrshl.s16 d2, d2, d30 |
| vext.8 q0, q0, q1, #2 |
| .else |
| mov r1, #4 |
| bl output_\lag\()_neon |
| .endif |
| .else |
| // elems == 1 |
| increment_seed 4, shift=0 |
| vmov q1, q2 |
| mov r1, #1 |
| bl output_\lag\()_neon |
| lsr r2, r2, #3 |
| |
| read_rand r11, 11, 2 |
| read_rand r12, 11, 1 |
| add r11, r3, r11, lsl #1 |
| add r12, r3, r12, lsl #1 |
| vld1.16 {d2[0]}, [r11] |
| read_rand r11, 11, 0 |
| vld1.16 {d2[1]}, [r12] |
| add r11, r3, r11, lsl #1 |
| vld1.16 {d2[2]}, [r11] |
| vrshl.s16 d2, d2, d30 |
| vext.8 q0, q0, q1, #14 |
| .endif |
| vst1.16 {q0}, [r0]! |
| pop {r11} |
| pop {r1, pc} |
| .endif |
| .endm |
| |
| .macro sum_lag1_func type, uv_layout, edge, elems=8 |
| function sum_\type\()_lag1_\edge\()_neon |
| push {r1, lr} |
| .ifc \edge, left |
| sub r12, r0, #1*GRAIN_WIDTH*2 |
| vld1.8 {q9}, [r12] // load the previous block right above |
| .endif |
| sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems |
| endfunc |
| .endm |
| |
| sum_lag1_func y, 0, left |
| sum_lag1_func y, 0, mid |
| sum_lag1_func y, 0, right, 7 |
| sum_lag1_func uv_444, 444, left |
| sum_lag1_func uv_444, 444, mid |
| sum_lag1_func uv_444, 444, right, 7 |
| sum_lag1_func uv_422, 422, left |
| sum_lag1_func uv_422, 422, mid |
| sum_lag1_func uv_422, 422, right, 1 |
| sum_lag1_func uv_420, 420, left |
| sum_lag1_func uv_420, 420, mid |
| sum_lag1_func uv_420, 420, right, 1 |
| |
| |
| function sum_lag2_above_neon |
| push {lr} |
| sub r12, r0, #2*GRAIN_WIDTH*2 - 16 |
| sub lr, r0, #1*GRAIN_WIDTH*2 - 16 |
| vld1.16 {q10}, [r12] // load top right |
| vld1.16 {q13}, [lr] |
| |
| vdup.8 d10, d28[0] |
| vext.8 q0, q8, q9, #12 // top left, top mid |
| vdup.8 d12, d28[1] |
| vext.8 q1, q8, q9, #14 |
| vdup.8 d14, d28[3] |
| vext.8 q4, q9, q10, #2 // top mid, top right |
| vmovl.s8 q5, d10 |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| |
| vmull.s16 q2, d0, d10 |
| vmlal.s16 q2, d2, d12 |
| vmlal.s16 q2, d8, d14 |
| vmull.s16 q3, d1, d10 |
| vmlal.s16 q3, d3, d12 |
| vmlal.s16 q3, d9, d14 |
| |
| vdup.8 d10, d28[4] |
| vext.8 q0, q9, q10, #4 // top mid, top right |
| vdup.8 d12, d28[5] |
| vext.8 q1, q11, q12, #12 // top left, top mid |
| vdup.8 d14, d28[6] |
| vext.8 q4, q11, q12, #14 |
| vmovl.s8 q5, d10 |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| |
| vmlal.s16 q2, d0, d10 |
| vmlal.s16 q2, d2, d12 |
| vmlal.s16 q2, d8, d14 |
| vmlal.s16 q3, d1, d10 |
| vmlal.s16 q3, d3, d12 |
| vmlal.s16 q3, d9, d14 |
| |
| vdup.8 d10, d29[0] |
| vext.8 q0, q12, q13, #2 // top mid, top right |
| vdup.8 d12, d29[1] |
| vext.8 q1, q12, q13, #4 |
| |
| vdup.8 d14, d28[2] |
| vdup.8 d8, d28[7] |
| |
| vmovl.s8 q5, d10 |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vmovl.s8 q4, d8 |
| |
| vmlal.s16 q2, d0, d10 |
| vmlal.s16 q2, d2, d12 |
| vmlal.s16 q2, d18, d14 |
| vmlal.s16 q2, d24, d8 |
| vmlal.s16 q3, d1, d10 |
| vmlal.s16 q3, d3, d12 |
| vmlal.s16 q3, d19, d14 |
| vmlal.s16 q3, d25, d8 |
| |
| vmov q8, q9 |
| vmov q9, q10 |
| |
| vmov q11, q12 |
| vmov q12, q13 |
| |
| pop {pc} |
| endfunc |
| |
| .macro sum_lag2_func type, uv_layout, edge, elems=8 |
| function sum_\type\()_lag2_\edge\()_neon |
| push {r1, lr} |
| .ifc \edge, left |
| sub r12, r0, #2*GRAIN_WIDTH*2 |
| sub lr, r0, #1*GRAIN_WIDTH*2 |
| vld1.16 {q9}, [r12] // load the previous block right above |
| vld1.16 {q12}, [lr] |
| .endif |
| sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, uv_coeff=d29[4] |
| endfunc |
| .endm |
| |
| sum_lag2_func y, 0, left |
| sum_lag2_func y, 0, mid |
| sum_lag2_func y, 0, right, 7 |
| sum_lag2_func uv_444, 444, left |
| sum_lag2_func uv_444, 444, mid |
| sum_lag2_func uv_444, 444, right, 7 |
| sum_lag2_func uv_422, 422, left |
| sum_lag2_func uv_422, 422, mid |
| sum_lag2_func uv_422, 422, right, 1 |
| sum_lag2_func uv_420, 420, left |
| sum_lag2_func uv_420, 420, mid |
| sum_lag2_func uv_420, 420, right, 1 |
| |
| |
| function sum_lag3_left_above_neon |
| // A separate codepath for the left edge, to avoid reading outside |
| // of the edge of the buffer. |
| sub r12, r0, #3*GRAIN_WIDTH*2 |
| vld1.8 {q11, q12}, [r12] |
| vext.8 q12, q11, q12, #10 |
| vext.8 q11, q11, q11, #10 |
| b sum_lag3_above_start |
| endfunc |
| |
| function sum_lag3_above_neon |
| movw r12, #(3*GRAIN_WIDTH + 3)*2 |
| sub r12, r0, r12 |
| vld1.8 {q11, q12}, [r12] |
| |
| sum_lag3_above_start: |
| vdup.8 d12, d26[0] |
| vext.8 q1, q11, q12, #2 |
| vdup.8 d14, d26[1] |
| vext.8 q4, q11, q12, #4 |
| vdup.8 d16, d26[2] |
| vext.8 q5, q11, q12, #6 |
| vdup.8 d18, d26[3] |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vmovl.s8 q8, d16 |
| vmovl.s8 q9, d18 |
| |
| movw r12, #(2*GRAIN_WIDTH + 3)*2 |
| sub r12, r0, r12 |
| |
| vmull.s16 q2, d22, d12 |
| vmlal.s16 q2, d2, d14 |
| vmlal.s16 q2, d8, d16 |
| vmlal.s16 q2, d10, d18 |
| vmull.s16 q3, d23, d12 |
| vmlal.s16 q3, d3, d14 |
| vmlal.s16 q3, d9, d16 |
| vmlal.s16 q3, d11, d18 |
| |
| vdup.8 d12, d26[4] |
| vext.8 q0, q11, q12, #8 |
| vdup.8 d14, d26[5] |
| vext.8 q1, q11, q12, #10 |
| vdup.8 d16, d26[6] |
| vext.8 q4, q11, q12, #12 |
| vld1.8 {q11, q12}, [r12] |
| vdup.8 d18, d26[7] |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vmovl.s8 q8, d16 |
| vmovl.s8 q9, d18 |
| |
| vmlal.s16 q2, d0, d12 |
| vmlal.s16 q2, d2, d14 |
| vmlal.s16 q2, d8, d16 |
| vmlal.s16 q2, d22, d18 |
| vmlal.s16 q3, d1, d12 |
| vmlal.s16 q3, d3, d14 |
| vmlal.s16 q3, d9, d16 |
| vmlal.s16 q3, d23, d18 |
| |
| vdup.8 d12, d27[0] |
| vext.8 q0, q11, q12, #2 |
| vdup.8 d14, d27[1] |
| vext.8 q1, q11, q12, #4 |
| vdup.8 d16, d27[2] |
| vext.8 q4, q11, q12, #6 |
| vdup.8 d18, d27[3] |
| vext.8 q5, q11, q12, #8 |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vmovl.s8 q8, d16 |
| vmovl.s8 q9, d18 |
| |
| sub r12, r0, #(1*GRAIN_WIDTH + 3)*2 |
| |
| vmlal.s16 q2, d0, d12 |
| vmlal.s16 q2, d2, d14 |
| vmlal.s16 q2, d8, d16 |
| vmlal.s16 q2, d10, d18 |
| vmlal.s16 q3, d1, d12 |
| vmlal.s16 q3, d3, d14 |
| vmlal.s16 q3, d9, d16 |
| vmlal.s16 q3, d11, d18 |
| |
| vdup.8 d12, d27[4] |
| vext.8 q0, q11, q12, #10 |
| vdup.8 d14, d27[5] |
| vext.8 q1, q11, q12, #12 |
| vld1.8 {q11, q12}, [r12] |
| vdup.8 d16, d27[6] |
| vdup.8 d18, d27[7] |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vext.8 q5, q11, q12, #2 |
| vmovl.s8 q8, d16 |
| vmovl.s8 q9, d18 |
| |
| vmlal.s16 q2, d0, d12 |
| vmlal.s16 q2, d2, d14 |
| vmlal.s16 q2, d22, d16 |
| vmlal.s16 q2, d10, d18 |
| vmlal.s16 q3, d1, d12 |
| vmlal.s16 q3, d3, d14 |
| vmlal.s16 q3, d23, d16 |
| vmlal.s16 q3, d11, d18 |
| |
| vdup.8 d12, d28[0] |
| vext.8 q0, q11, q12, #4 |
| vdup.8 d14, d28[1] |
| vext.8 q1, q11, q12, #6 |
| vdup.8 d16, d28[2] |
| vext.8 q4, q11, q12, #8 |
| vdup.8 d18, d28[3] |
| vext.8 q5, q11, q12, #10 |
| vmovl.s8 q6, d12 |
| vmovl.s8 q7, d14 |
| vmovl.s8 q8, d16 |
| vmovl.s8 q9, d18 |
| |
| vmlal.s16 q2, d0, d12 |
| vmlal.s16 q2, d2, d14 |
| vmlal.s16 q2, d8, d16 |
| vmlal.s16 q2, d10, d18 |
| vmlal.s16 q3, d1, d12 |
| vmlal.s16 q3, d3, d14 |
| vmlal.s16 q3, d9, d16 |
| vmlal.s16 q3, d11, d18 |
| |
| vdup.8 d12, d28[4] |
| vext.8 q0, q11, q12, #12 |
| vmovl.s8 q6, d12 |
| |
| vmlal.s16 q2, d0, d12 |
| vmlal.s16 q3, d1, d12 |
| |
| bx lr |
| endfunc |
| |
| .macro sum_lag3_func type, uv_layout, edge, elems=8 |
| function sum_\type\()_lag3_\edge\()_neon |
| push {r1, lr} |
| sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, uv_coeff=d29[0] |
| endfunc |
| .endm |
| |
| sum_lag3_func y, 0, left |
| sum_lag3_func y, 0, mid |
| sum_lag3_func y, 0, right, 7 |
| sum_lag3_func uv_444, 444, left |
| sum_lag3_func uv_444, 444, mid |
| sum_lag3_func uv_444, 444, right, 7 |
| sum_lag3_func uv_422, 422, left |
| sum_lag3_func uv_422, 422, mid |
| sum_lag3_func uv_422, 422, right, 1 |
| sum_lag3_func uv_420, 420, left |
| sum_lag3_func uv_420, 420, mid |
| sum_lag3_func uv_420, 420, right, 1 |
| |
| function generate_grain_rows_neon |
| push {r10-r11,lr} |
| 1: |
| mov r10, #80 |
| 2: |
| bl get_gaussian_neon |
| vrshl.s16 q0, q0, q15 |
| subs r10, r10, #8 |
| vst1.16 {q0}, [r0]! |
| bgt 2b |
| get_grain_2 d0 |
| subs r1, r1, #1 |
| vst1.32 {d0[0]}, [r0]! |
| bgt 1b |
| pop {r10-r11,pc} |
| endfunc |
| |
| function generate_grain_rows_44_neon |
| push {r10-r11,lr} |
| 1: |
| mov r10, #40 |
| 2: |
| bl get_gaussian_neon |
| vrshl.s16 q0, q0, q15 |
| subs r10, r10, #8 |
| vst1.16 {q0}, [r0]! |
| bgt 2b |
| get_grain_4 d0 |
| subs r1, r1, #1 |
| vst1.16 {d0}, [r0] |
| add r0, r0, #GRAIN_WIDTH*2-80 |
| bgt 1b |
| pop {r10-r11,pc} |
| endfunc |
| |
| function gen_grain_uv_444_lag0_neon |
| vld1.16 {q3}, [r11]! |
| gen_grain_uv_lag0_8_start: |
| push {r11,lr} |
| bl get_gaussian_neon |
| vrshl.s16 q0, q0, q15 |
| gen_grain_uv_lag0_8_add: |
| vand q3, q3, q1 |
| vmull.s16 q2, d6, d22 |
| vmull.s16 q3, d7, d22 |
| vrshl.s32 q2, q2, q12 |
| vrshl.s32 q3, q3, q12 |
| vqmovn.s32 d4, q2 |
| vqmovn.s32 d5, q3 |
| vqadd.s16 q2, q2, q0 |
| vmin.s16 q2, q2, q9 |
| vmax.s16 q2, q2, q10 |
| vst1.16 {q2}, [r0]! |
| pop {r11,pc} |
| endfunc |
| |
| function gen_grain_uv_420_lag0_8_neon |
| add r12, r11, #GRAIN_WIDTH*2 |
| vld1.16 {q2,q3}, [r11]! |
| vld1.16 {q4,q5}, [r12] |
| vpadd.i16 d4, d4, d5 |
| vpadd.i16 d5, d6, d7 |
| vpadd.i16 d8, d8, d9 |
| vpadd.i16 d9, d10, d11 |
| vadd.i16 q2, q2, q4 |
| vrshr.s16 q3, q2, #2 |
| b gen_grain_uv_lag0_8_start |
| endfunc |
| |
| function gen_grain_uv_422_lag0_8_neon |
| vld1.16 {q2,q3}, [r11]! |
| vpadd.i16 d4, d4, d5 |
| vpadd.i16 d5, d6, d7 |
| vrshr.s16 q3, q2, #1 |
| b gen_grain_uv_lag0_8_start |
| endfunc |
| |
| function gen_grain_uv_420_lag0_4_neon |
| add r12, r11, #GRAIN_WIDTH*2 |
| vld1.16 {q2}, [r11] |
| vld1.16 {q0}, [r12] |
| add r11, r11, #32 |
| vpadd.i16 d4, d4, d5 |
| vpadd.i16 d0, d0, d1 |
| vadd.i16 d4, d4, d0 |
| vrshr.s16 d6, d4, #2 |
| push {r11,lr} |
| get_grain_4 d0 |
| b gen_grain_uv_lag0_8_add |
| endfunc |
| |
| function gen_grain_uv_422_lag0_4_neon |
| vld1.16 {q2}, [r11] |
| add r11, r11, #32 |
| vpadd.i16 d4, d4, d5 |
| vrshr.s16 d6, d4, #1 |
| push {r11,lr} |
| get_grain_4 d0 |
| b gen_grain_uv_lag0_8_add |
| endfunc |
| |
| .macro gen_grain_82 type |
| function generate_grain_\type\()_16bpc_neon, export=1 |
| push {r4-r11,lr} |
| |
| .ifc \type, uv_444 |
| ldr r4, [sp, #36] |
| mov r12, r3 |
| mov lr, #28 |
| add r11, r1, #3*GRAIN_WIDTH*2 |
| mov r1, r2 |
| mul r12, r12, lr |
| clz lr, r4 |
| .else |
| clz lr, r2 |
| .endif |
| movrel r3, X(gaussian_sequence) |
| sub lr, lr, #24 // -bitdepth_min_8 |
| ldr r2, [r1, #FGD_SEED] |
| ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] |
| .ifc \type, y |
| add r4, r1, #FGD_AR_COEFFS_Y |
| .else |
| add r4, r1, #FGD_AR_COEFFS_UV |
| .endif |
| add r9, r9, lr // grain_scale_shift - bitdepth_min_8 |
| adr r5, L(gen_grain_\type\()_tbl) |
| ldr r6, [r1, #FGD_AR_COEFF_LAG] |
| add r9, r9, #4 |
| ldr r6, [r5, r6, lsl #2] |
| vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift |
| add r5, r5, r6 |
| vneg.s16 q15, q15 |
| |
| .ifc \type, uv_444 |
| push {lr} |
| cmp r12, #0 |
| movw r10, #0x49d8 |
| movw lr, #0xb524 |
| // Intentionally using a separate register instead of moveq with an |
| // immediate constant, to avoid armv8 deprecated it instruction forms. |
| it eq |
| moveq r10, lr |
| add r4, r4, r12 // Add offset to ar_coeffs_uv[1] |
| eor r2, r2, r10 |
| pop {lr} |
| .endif |
| |
| ldr r7, [r1, #FGD_AR_COEFF_SHIFT] |
| neg lr, lr // bitdepth_min_8 |
| mov r8, #1 |
| mov r10, #1 |
| lsl r8, r8, r7 // 1 << ar_coeff_shift |
| lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) |
| lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) |
| lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) |
| |
| bx r5 |
| |
| .align 2 |
| L(gen_grain_\type\()_tbl): |
| .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| |
| L(generate_grain_\type\()_lag0): |
| .ifc \type, y |
| mov r1, #GRAIN_HEIGHT |
| bl generate_grain_rows_neon |
| .else |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| mvn r6, r5 // grain_min = ~grain_max |
| |
| mov r1, #3 |
| bl generate_grain_rows_neon |
| mov r1, #GRAIN_HEIGHT-3 |
| |
| vdup.32 q12, r7 |
| vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] |
| vmov.i8 q0, #0 |
| vmov.i8 q1, #255 |
| vdup.16 q9, r5 |
| vdup.16 q10, r6 |
| vext.8 q13, q0, q1, #10 |
| vext.8 q14, q1, q0, #2 |
| vneg.s32 q12, q12 |
| vmovl.s8 q11, d22 |
| |
| 1: |
| vmov q1, q13 |
| bl gen_grain_uv_444_lag0_neon // 8 |
| vmov.i8 q1, #255 |
| bl gen_grain_uv_444_lag0_neon // 16 |
| bl gen_grain_uv_444_lag0_neon // 24 |
| bl gen_grain_uv_444_lag0_neon // 32 |
| bl gen_grain_uv_444_lag0_neon // 40 |
| bl gen_grain_uv_444_lag0_neon // 48 |
| bl gen_grain_uv_444_lag0_neon // 56 |
| bl gen_grain_uv_444_lag0_neon // 64 |
| bl gen_grain_uv_444_lag0_neon // 72 |
| vmov q1, q14 |
| bl gen_grain_uv_444_lag0_neon // 80 |
| get_grain_2 d16 |
| subs r1, r1, #1 |
| add r11, r11, #4 |
| vst1.32 {d16[0]}, [r0]! |
| bgt 1b |
| .endif |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag1): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] |
| vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] |
| vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] |
| .ifc \type, y |
| ldrsb r4, [r4, #1] // ar_coeffs_y[3] |
| .else |
| add r4, r4, #2 |
| .endif |
| |
| mov r1, #3 |
| .ifc \type, uv_444 |
| vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] |
| ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] |
| .endif |
| bl generate_grain_rows_neon |
| vmovl.s8 q13, d27 |
| vmovl.s8 q12, d29 |
| vmovl.s8 q14, d28 |
| vmov d29, d24 |
| .ifc \type, uv_444 |
| vmovl.s8 q6, d13 |
| .endif |
| |
| mov r1, #GRAIN_HEIGHT - 3 |
| 1: |
| bl sum_\type\()_lag1_left_neon // 8 |
| bl sum_\type\()_lag1_mid_neon // 16 |
| bl sum_\type\()_lag1_mid_neon // 24 |
| bl sum_\type\()_lag1_mid_neon // 32 |
| bl sum_\type\()_lag1_mid_neon // 40 |
| bl sum_\type\()_lag1_mid_neon // 48 |
| bl sum_\type\()_lag1_mid_neon // 56 |
| bl sum_\type\()_lag1_mid_neon // 64 |
| bl sum_\type\()_lag1_mid_neon // 72 |
| bl sum_\type\()_lag1_right_neon // 80 |
| get_grain_2 d16 |
| subs r1, r1, #1 |
| .ifc \type, uv_444 |
| add r11, r11, #4 |
| .endif |
| vst1.32 {d16[0]}, [r0]! |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag2): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] |
| |
| vmov.s8 r4, d29[2] |
| vmov.s8 r10, d29[3] |
| |
| mov r1, #3 |
| bl generate_grain_rows_neon |
| |
| mov r1, #GRAIN_HEIGHT - 3 |
| 1: |
| bl sum_\type\()_lag2_left_neon // 8 |
| bl sum_\type\()_lag2_mid_neon // 16 |
| bl sum_\type\()_lag2_mid_neon // 24 |
| bl sum_\type\()_lag2_mid_neon // 32 |
| bl sum_\type\()_lag2_mid_neon // 40 |
| bl sum_\type\()_lag2_mid_neon // 48 |
| bl sum_\type\()_lag2_mid_neon // 56 |
| bl sum_\type\()_lag2_mid_neon // 64 |
| bl sum_\type\()_lag2_mid_neon // 72 |
| bl sum_\type\()_lag2_right_neon // 80 |
| get_grain_2 d16 |
| subs r1, r1, #1 |
| .ifc \type, uv_444 |
| add r11, r11, #4 |
| .endif |
| vst1.32 {d16[0]}, [r0]! |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag3): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] |
| |
| vmov.u8 r4, d28[5] |
| vmov.u8 r10, d28[6] |
| vmov.u8 r12, d28[7] |
| |
| orr r4, r4, r10, lsl #8 |
| orr r4, r4, r12, lsl #16 |
| |
| mov r1, #3 |
| vpush {d26} |
| bl generate_grain_rows_neon |
| vpop {d26} |
| |
| mov r1, #GRAIN_HEIGHT - 3 |
| 1: |
| bl sum_\type\()_lag3_left_neon // 8 |
| bl sum_\type\()_lag3_mid_neon // 16 |
| bl sum_\type\()_lag3_mid_neon // 24 |
| bl sum_\type\()_lag3_mid_neon // 32 |
| bl sum_\type\()_lag3_mid_neon // 40 |
| bl sum_\type\()_lag3_mid_neon // 48 |
| bl sum_\type\()_lag3_mid_neon // 56 |
| bl sum_\type\()_lag3_mid_neon // 64 |
| bl sum_\type\()_lag3_mid_neon // 72 |
| bl sum_\type\()_lag3_right_neon // 80 |
| get_grain_2 d16 |
| subs r1, r1, #1 |
| .ifc \type, uv_444 |
| add r11, r11, #4 |
| .endif |
| vst1.32 {d16[0]}, [r0]! |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| endfunc |
| .endm |
| |
| gen_grain_82 y |
| gen_grain_82 uv_444 |
| |
| .macro set_height dst, type |
| .ifc \type, uv_420 |
| mov \dst, #SUB_GRAIN_HEIGHT-3 |
| .else |
| mov \dst, #GRAIN_HEIGHT-3 |
| .endif |
| .endm |
| |
| .macro increment_y_ptr reg, type |
| .ifc \type, uv_420 |
| add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) |
| .else |
| sub \reg, \reg, #6*32-GRAIN_WIDTH*2 |
| .endif |
| .endm |
| |
| .macro gen_grain_44 type |
| function generate_grain_\type\()_16bpc_neon, export=1 |
| push {r4-r11,lr} |
| |
| ldr r4, [sp, #36] |
| mov r12, r3 |
| movw r11, #(3*GRAIN_WIDTH-3)*2 |
| mov lr, #28 |
| add r11, r1, r11 |
| mov r1, r2 |
| mul r12, r12, lr |
| clz lr, r4 |
| |
| movrel r3, X(gaussian_sequence) |
| sub lr, lr, #24 // -bitdepth_min_8 |
| ldr r2, [r1, #FGD_SEED] |
| ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] |
| add r4, r1, #FGD_AR_COEFFS_UV |
| add r9, r9, lr // grain_scale_shift - bitdepth_min_8 |
| adr r5, L(gen_grain_\type\()_tbl) |
| ldr r6, [r1, #FGD_AR_COEFF_LAG] |
| add r9, r9, #4 |
| ldr r6, [r5, r6, lsl #2] |
| vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift |
| add r5, r5, r6 |
| vneg.s16 q15, q15 |
| |
| push {lr} |
| cmp r12, #0 |
| movw r10, #0x49d8 |
| movw lr, #0xb524 |
| // Intentionally using a separate register instead of moveq with an |
| // immediate constant, to avoid armv8 deprecated it instruction forms. |
| it eq |
| moveq r10, lr |
| add r4, r4, r12 // Add offset to ar_coeffs_uv[1] |
| eor r2, r2, r10 |
| pop {lr} |
| |
| ldr r7, [r1, #FGD_AR_COEFF_SHIFT] |
| neg lr, lr |
| mov r8, #1 |
| mov r10, #1 |
| lsl r8, r8, r7 // 1 << ar_coeff_shift |
| lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) |
| lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) |
| lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) |
| bx r5 |
| |
| .align 2 |
| L(gen_grain_\type\()_tbl): |
| .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB |
| |
| L(generate_grain_\type\()_lag0): |
| .ifc \type, uv_420 |
| vpush {q4-q5} |
| .endif |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| mvn r6, r5 // grain_min = ~grain_max |
| |
| mov r1, #3 |
| bl generate_grain_rows_44_neon |
| set_height r1, \type |
| |
| vdup.32 q12, r7 |
| vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] |
| vmov.i8 q0, #0 |
| vmov.i8 q1, #255 |
| vdup.16 q9, r5 |
| vdup.16 q10, r6 |
| vext.8 q13, q0, q1, #10 |
| vext.8 q14, q1, q0, #14 |
| vneg.s32 q12, q12 |
| vmovl.s8 q11, d22 |
| |
| 1: |
| vmov q1, q13 |
| bl gen_grain_\type\()_lag0_8_neon // 8 |
| vmov.i8 q1, #255 |
| bl gen_grain_\type\()_lag0_8_neon // 16 |
| bl gen_grain_\type\()_lag0_8_neon // 24 |
| bl gen_grain_\type\()_lag0_8_neon // 32 |
| bl gen_grain_\type\()_lag0_8_neon // 40 |
| vmov q1, q14 |
| bl gen_grain_\type\()_lag0_4_neon // 44 |
| subs r1, r1, #1 |
| increment_y_ptr r11, \type |
| add r0, r0, #GRAIN_WIDTH*2-6*16 |
| bgt 1b |
| |
| .ifc \type, uv_420 |
| vpop {q4-q5} |
| .endif |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag1): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] |
| vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] |
| vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] |
| add r4, r4, #2 |
| |
| mov r1, #3 |
| vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] |
| ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] |
| bl generate_grain_rows_44_neon |
| vmovl.s8 q13, d27 |
| vmovl.s8 q12, d29 |
| vmovl.s8 q14, d28 |
| vmov d29, d24 |
| vmovl.s8 q6, d13 |
| |
| set_height r1, \type |
| 1: |
| bl sum_\type\()_lag1_left_neon // 8 |
| bl sum_\type\()_lag1_mid_neon // 16 |
| bl sum_\type\()_lag1_mid_neon // 24 |
| bl sum_\type\()_lag1_mid_neon // 32 |
| bl sum_\type\()_lag1_mid_neon // 40 |
| bl sum_\type\()_lag1_right_neon // 44 |
| subs r1, r1, #1 |
| increment_y_ptr r11, \type |
| add r0, r0, #GRAIN_WIDTH*2-6*16 |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag2): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] |
| |
| vmov.s8 r4, d29[2] |
| vmov.s8 r10, d29[3] |
| |
| mov r1, #3 |
| bl generate_grain_rows_44_neon |
| |
| set_height r1, \type |
| 1: |
| bl sum_\type\()_lag2_left_neon // 8 |
| bl sum_\type\()_lag2_mid_neon // 16 |
| bl sum_\type\()_lag2_mid_neon // 24 |
| bl sum_\type\()_lag2_mid_neon // 32 |
| bl sum_\type\()_lag2_mid_neon // 40 |
| bl sum_\type\()_lag2_right_neon // 44 |
| subs r1, r1, #1 |
| increment_y_ptr r11, \type |
| add r0, r0, #GRAIN_WIDTH*2-6*16 |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| |
| L(generate_grain_\type\()_lag3): |
| vpush {q4-q7} |
| mov r5, #128 |
| lsl r5, r5, lr // 128 << bitdepth_min_8 |
| sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 |
| vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] |
| |
| vmov.u8 r4, d28[5] |
| vmov.u8 r10, d28[6] |
| vmov.u8 r12, d28[7] |
| |
| orr r4, r4, r10, lsl #8 |
| orr r4, r4, r12, lsl #16 |
| |
| mov r1, #3 |
| bl generate_grain_rows_44_neon |
| |
| set_height r1, \type |
| 1: |
| bl sum_\type\()_lag3_left_neon // 8 |
| bl sum_\type\()_lag3_mid_neon // 16 |
| bl sum_\type\()_lag3_mid_neon // 24 |
| bl sum_\type\()_lag3_mid_neon // 32 |
| bl sum_\type\()_lag3_mid_neon // 40 |
| bl sum_\type\()_lag3_right_neon // 44 |
| subs r1, r1, #1 |
| increment_y_ptr r11, \type |
| add r0, r0, #GRAIN_WIDTH*2-6*16 |
| bgt 1b |
| |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| endfunc |
| .endm |
| |
| gen_grain_44 uv_420 |
| gen_grain_44 uv_422 |
| |
| .macro gather_interleaved dst1, dst2, src1, src2, src3, src4, off |
| vmov.u16 r11, \src1[0+\off] |
| vmov.u16 r12, \src3[0+\off] |
| add r11, r11, r3 |
| vmov.u16 lr, \src1[2+\off] |
| add r12, r12, r3 |
| vld1.8 {\dst1[0+\off]}, [r11] |
| vmov.u16 r11, \src3[2+\off] |
| add lr, lr, r3 |
| vld1.8 {\dst2[0+\off]}, [r12] |
| vmov.u16 r12, \src2[0+\off] |
| add r11, r11, r3 |
| vld1.8 {\dst1[2+\off]}, [lr] |
| vmov.u16 lr, \src4[0+\off] |
| add r12, r12, r3 |
| vld1.8 {\dst2[2+\off]}, [r11] |
| vmov.u16 r11, \src2[2+\off] |
| add lr, lr, r3 |
| vld1.8 {\dst1[4+\off]}, [r12] |
| vmov.u16 r12, \src4[2+\off] |
| add r11, r11, r3 |
| vld1.8 {\dst2[4+\off]}, [lr] |
| add r12, r12, r3 |
| vld1.8 {\dst1[6+\off]}, [r11] |
| vld1.8 {\dst2[6+\off]}, [r12] |
| .endm |
| |
| .macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, src7, src8 |
| gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 0 |
| gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 1 |
| gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 0 |
| gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 1 |
| .endm |
| |
| function gather32_neon |
| push {r11-r12,lr} |
| gather d8, d9, d10, d11, d0, d1, d2, d3, d4, d5, d6, d7 |
| pop {r11-r12,pc} |
| endfunc |
| |
| function gather16_neon |
| push {r11-r12,lr} |
| gather_interleaved d8, d9, d0, d1, d2, d3, 0 |
| gather_interleaved d8, d9, d0, d1, d2, d3, 1 |
| pop {r11-r12,pc} |
| endfunc |
| |
| const overlap_coeffs_0, align=4 |
| .short 27, 17, 0, 0 |
| .short 17, 27, 32, 32 |
| endconst |
| |
| const overlap_coeffs_1, align=4 |
| .short 23, 0, 0, 0 |
| .short 22, 32, 32, 32 |
| endconst |
| |
| .macro calc_offset offx, offy, src, sx, sy |
| and \offy, \src, #0xF // randval & 0xF |
| lsr \offx, \src, #4 // randval >> 4 |
| .if \sy == 0 |
| add \offy, \offy, \offy // 2 * (randval & 0xF) |
| .endif |
| .if \sx == 0 |
| add \offx, \offx, \offx // 2 * (randval >> 4) |
| .endif |
| .endm |
| |
| .macro add_offset dst, offx, offy, src, stride |
| mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy |
| add \dst, \dst, \offx, lsl #1 // grain_lut += offx |
| .endm |
| |
| // void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src, |
| // const ptrdiff_t stride, |
| // const uint8_t scaling[SCALING_SIZE], |
| // const int scaling_shift, |
| // const entry grain_lut[][GRAIN_WIDTH], |
| // const int offsets[][2], |
| // const int h, const ptrdiff_t clip, |
| // const ptrdiff_t type, |
| // const int bitdepth_max); |
| function fgy_32x32_16bpc_neon, export=1 |
| push {r4-r11,lr} |
| vpush {q4-q7} |
| ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut |
| ldrd r6, r7, [sp, #108] // offsets, h |
| ldr r8, [sp, #116] // clip |
| mov r9, #GRAIN_WIDTH*2 // grain_lut stride |
| ldr r10, [sp, #124] // bitdepth_max |
| |
| eor r4, r4, #15 // 15 - scaling_shift |
| vdup.16 q6, r10 // bitdepth_max |
| clz r10, r10 |
| vdup.16 q13, r4 // 15 - scaling_shift |
| rsb r10, r10, #24 // bitdepth_min_8 |
| cmp r8, #0 |
| vdup.16 q12, r10 // bitdepth_min_8 |
| |
| movrel_local r12, overlap_coeffs_0 |
| |
| beq 1f |
| // clip |
| vmov.i16 q14, #16 |
| vmov.i16 q15, #235 |
| vshl.s16 q14, q14, q12 |
| vshl.s16 q15, q15, q12 |
| b 2f |
| 1: |
| // no clip |
| vmov.i16 q14, #0 |
| vmov q15, q6 |
| 2: |
| vshr.u16 q6, q6, #1 // grain_max |
| |
| vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs |
| |
| add r5, r5, #18 // grain_lut += 9 |
| add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride |
| add r5, r5, r9 // grain_lut += grain_stride |
| |
| ldr r10, [r6, #8] // offsets[1][0] |
| calc_offset r10, r4, r10, 0, 0 |
| add_offset r4, r10, r4, r5, r9 |
| ldr r10, [r6, #4] // offsets[0][1] |
| calc_offset r10, r11, r10, 0, 0 |
| add_offset r11, r10, r11, r5, r9 |
| ldr r10, [r6, #12] // offsets[1][1] |
| calc_offset r10, r8, r10, 0, 0 |
| add_offset r8, r10, r8, r5, r9 |
| ldr r6, [r6] // offsets[0][0] |
| calc_offset r6, lr, r6, 0, 0 |
| add_offset r5, r6, lr, r5, r9 |
| |
| add r4, r4, #32*2 // grain_lut += BLOCK_SIZE * bx |
| add r6, r11, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by |
| |
| ldr r10, [sp, #120] // type |
| adr r11, L(fgy_loop_tbl) |
| |
| tst r10, #1 |
| ldr r10, [r11, r10, lsl #2] |
| |
| add r8, r8, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by |
| add r8, r8, #32*2 // grain_lut += BLOCK_SIZE * bx |
| |
| add r11, r11, r10 |
| |
| beq 1f |
| // y overlap |
| vdup.16 d14, d24[0] |
| vdup.16 d15, d24[1] |
| mov r10, r7 // backup actual h |
| mov r7, #2 |
| 1: |
| sub r2, r2, #32 // src_stride -= 32 |
| sub r9, r9, #32 // grain_stride -= 32 |
| bx r11 |
| endfunc |
| |
| function fgy_loop_neon |
| L(fgy_loop_tbl): |
| .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB |
| .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB |
| .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB |
| .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB |
| |
| .macro fgy ox, oy |
| L(loop_\ox\oy): |
| 1: |
| .if \ox |
| vld1.16 {d0}, [r4], r9 // grain_lut old |
| .endif |
| .if \oy |
| vld1.16 {q2, q3}, [r6]! // grain_lut top |
| .endif |
| .if \ox && \oy |
| vld1.16 {d2}, [r8], r9 // grain_lut top old |
| .endif |
| .if \oy |
| vld1.16 {q4, q5}, [r6], r9 // grain_lut top |
| .endif |
| .if !\ox && !\oy |
| vld1.16 {q0, q1}, [r1, :128]! // src |
| .endif |
| vld1.16 {q8, q9}, [r5]! // grain_lut |
| .if !\ox && !\oy |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| .endif |
| .if !\oy |
| vmvn.i16 q5, #0xf000 // 0x0fff |
| .endif |
| vld1.16 {q10, q11}, [r5], r9 // grain_lut |
| |
| .if \ox |
| add r4, r4, #32 |
| vmull.s16 q0, d0, d24 |
| vmlal.s16 q0, d16, d25 |
| .endif |
| |
| .if \oy |
| .if \ox |
| add r8, r8, #32 |
| vmull.s16 q1, d2, d24 |
| vmlal.s16 q1, d4, d25 |
| vqrshrn.s32 d16, q0, #5 |
| vmvn d0, d12 // grain_min |
| vqrshrn.s32 d4, q1, #5 |
| vmin.s16 d16, d16, d12 |
| vmin.s16 d4, d4, d12 |
| vmax.s16 d16, d16, d0 |
| vmax.s16 d4, d4, d0 |
| .endif |
| |
| vmull.s16 q0, d4, d14 |
| vmull.s16 q1, d5, d14 |
| vmull.s16 q2, d6, d14 |
| vmull.s16 q3, d7, d14 |
| vmlal.s16 q0, d16, d15 |
| vmlal.s16 q1, d17, d15 |
| vmlal.s16 q2, d18, d15 |
| vmlal.s16 q3, d19, d15 |
| vmull.s16 q8, d20, d15 |
| vmull.s16 q9, d21, d15 |
| vmull.s16 q10, d22, d15 |
| vmull.s16 q11, d23, d15 |
| vmlal.s16 q8, d8, d14 |
| vmlal.s16 q9, d9, d14 |
| vmlal.s16 q10, d10, d14 |
| vmlal.s16 q11, d11, d14 |
| vmvn q4, q6 // grain_min |
| vqrshrn.s32 d0, q0, #5 |
| vqrshrn.s32 d1, q1, #5 |
| vqrshrn.s32 d2, q2, #5 |
| vqrshrn.s32 d3, q3, #5 |
| vqrshrn.s32 d4, q8, #5 |
| vqrshrn.s32 d5, q9, #5 |
| vqrshrn.s32 d6, q10, #5 |
| vqrshrn.s32 d7, q11, #5 |
| vmin.s16 q8, q0, q6 |
| vmin.s16 q9, q1, q6 |
| vld1.16 {q0, q1}, [r1, :128]! // src |
| vmin.s16 q10, q2, q6 |
| vmin.s16 q11, q3, q6 |
| vmax.s16 q8, q8, q4 |
| vmax.s16 q9, q9, q4 |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| vmvn.i16 q5, #0xf000 // 0x0fff |
| vmax.s16 q10, q10, q4 |
| vmax.s16 q11, q11, q4 |
| .elseif \ox |
| vmvn d4, d12 // grain_min |
| vqrshrn.s32 d16, q0, #5 |
| vld1.16 {q0, q1}, [r1, :128]! // src |
| vmin.s16 d16, d16, d12 |
| vmax.s16 d16, d16, d4 |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| .endif |
| |
| // Make sure that uninitialized pixels out of range past the right |
| // edge are in range; their actual values shouldn't matter. |
| vand q0, q0, q5 |
| vand q1, q1, q5 |
| vand q2, q2, q5 |
| vand q3, q3, q5 |
| |
| bl gather32_neon |
| |
| .if \ox || \oy |
| vpush {q6-q7} |
| .endif |
| |
| vmovl.u8 q6, d8 // scaling |
| vmovl.u8 q7, d9 |
| vmovl.u8 q4, d10 |
| vmovl.u8 q5, d11 |
| |
| vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) |
| vshl.u16 q7, q7, q13 |
| vshl.u16 q4, q4, q13 |
| vshl.u16 q5, q5, q13 |
| |
| vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) |
| vqrdmulh.s16 q9, q9, q7 |
| vqrdmulh.s16 q10, q10, q4 |
| vqrdmulh.s16 q11, q11, q5 |
| |
| .if \ox || \oy |
| vpop {q6-q7} |
| .endif |
| |
| vqadd.s16 q0, q0, q8 // *src + noise |
| vqadd.s16 q1, q1, q9 |
| vqadd.s16 q2, q2, q10 |
| vqadd.s16 q3, q3, q11 |
| |
| vmax.s16 q0, q0, q14 |
| vmax.s16 q1, q1, q14 |
| vmax.s16 q2, q2, q14 |
| vmax.s16 q3, q3, q14 |
| vmin.s16 q0, q0, q15 |
| vmin.s16 q1, q1, q15 |
| vmin.s16 q2, q2, q15 |
| vmin.s16 q3, q3, q15 |
| |
| vst1.16 {q0, q1}, [r0, :128]! // dst |
| subs r7, r7, #1 |
| .if \oy |
| vdup.16 d14, d25[0] |
| vdup.16 d15, d25[1] |
| .endif |
| vst1.16 {q2, q3}, [r0, :128], r2 // dst |
| bgt 1b |
| |
| .if \oy |
| cmp r10, #2 |
| sub r7, r10, #2 // restore actual remaining h |
| bgt L(loop_\ox\()0) |
| .endif |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| .endm |
| |
| fgy 0, 0 |
| fgy 0, 1 |
| fgy 1, 0 |
| fgy 1, 1 |
| endfunc |
| |
| // void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst, |
| // const pixel *const src, |
| // const ptrdiff_t stride, |
| // const uint8_t scaling[SCALING_SIZE], |
| // const Dav1dFilmGrainData *const data, |
| // const entry grain_lut[][GRAIN_WIDTH], |
| // const pixel *const luma_row, |
| // const ptrdiff_t luma_stride, |
| // const int offsets[][2], |
| // const ptrdiff_t h, const ptrdiff_t uv, |
| // const ptrdiff_t is_id, |
| // const ptrdiff_t type, |
| // const int bitdepth_max); |
| .macro fguv layout, sx, sy |
| function fguv_32x32_\layout\()_16bpc_neon, export=1 |
| push {r4-r11,lr} |
| vpush {q4-q7} |
| ldrd r4, r5, [sp, #100] // data, grain_lut |
| ldrd r10, r11, [sp, #124] // uv, is_id |
| ldr r6, [sp, #136] // bitdepth_max |
| |
| clz r7, r6 |
| rsb r7, r7, #24 // bitdepth_min_8 |
| |
| // !csfl |
| add r10, r4, r10, lsl #2 // + 4*uv |
| add r12, r10, #FGD_UV_LUMA_MULT |
| add lr, r10, #FGD_UV_MULT |
| ldrh r10, [r10, #FGD_UV_OFFSET] // uv_offset |
| vld1.16 {d30[]}, [r12] // uv_luma_mult |
| lsl r10, r10, r7 // uv_offset << bitdepth_min_8 |
| vld1.16 {d30[1]}, [lr] // uv_mult |
| |
| ldr lr, [r4, #FGD_SCALING_SHIFT] |
| ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE] |
| eor lr, lr, #15 // 15 - scaling_shift |
| |
| vmov.16 d30[2], r10 // uv_offset << bitdepth_min_8 |
| |
| cmp r12, #0 |
| vdup.16 q13, lr // 15 - scaling_shift |
| |
| beq 1f |
| // clip |
| cmp r11, #0 |
| mov r8, #16 |
| mov r9, #240 |
| lsl r8, r8, r7 |
| lsl r9, r9, r7 |
| beq 2f |
| // is_id |
| mov r9, #235 |
| lsl r9, r9, r7 |
| b 2f |
| 1: |
| // no clip |
| mov r8, #0 |
| mov r9, r6 // bitdepth_max |
| 2: |
| vmov.16 d30[3], r6 // bitdepth_max |
| vdup.16 d31, r8 // clip_min |
| |
| mov r10, #GRAIN_WIDTH*2 // grain_lut stride |
| |
| .if \sy |
| mov r6, #23 |
| mov r7, #22 |
| .else |
| mov r6, #27 |
| mov r7, #17 |
| .endif |
| vmov.16 d31[1], r9 // clip_max |
| |
| ldrd r8, r9, [sp, #116] // offsets, h |
| |
| add r5, r5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6 |
| .if \sy |
| add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride |
| add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride |
| .else |
| add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride |
| add r5, r5, r10 // grain_lut += grain_stride |
| .endif |
| vmov.16 d31[2], r6 // overlap y [0] |
| |
| ldr r12, [r8, #8] // offsets[1][0] |
| calc_offset r12, r4, r12, \sx, \sy |
| add_offset r4, r12, r4, r5, r10 |
| |
| ldr r12, [r8, #4] // offsets[0][1] |
| calc_offset r12, lr, r12, \sx, \sy |
| add_offset lr, r12, lr, r5, r10 |
| |
| ldr r12, [r8, #12] // offsets[1][1] |
| calc_offset r12, r11, r12, \sx, \sy |
| add_offset r11, r12, r11, r5, r10 |
| |
| ldr r8, [r8] // offsets[0][0] |
| calc_offset r8, r12, r8, \sx, \sy |
| add_offset r5, r8, r12, r5, r10 |
| |
| vmov.16 d31[3], r7 // overlap y [1] |
| |
| add r4, r4, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx |
| add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by |
| add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by |
| add r11, r11, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx |
| |
| movrel_local r12, overlap_coeffs_\sx |
| ldr lr, [sp, #132] // type |
| ldrd r6, r7, [sp, #108] // luma_row, luma_stride |
| |
| vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs |
| |
| movrel_local r12, L(fguv_loop_sx\sx\()_tbl) |
| #if CONFIG_THUMB |
| // This uses movrel_local instead of adr above, because the target |
| // can be out of range for adr. But movrel_local leaves the thumb bit |
| // set on COFF (but probably wouldn't if building for thumb on ELF), |
| // thus try to clear the bit for robustness. |
| bic r12, r12, #1 |
| #endif |
| |
| tst lr, #1 |
| ldr lr, [r12, lr, lsl #2] |
| |
| add r12, r12, lr |
| |
| beq 1f |
| // y overlap |
| sub lr, r9, #(2 >> \sy) // backup remaining h |
| mov r9, #(2 >> \sy) |
| |
| 1: |
| .if \sy |
| add r7, r7, r7 // luma_stride *= 2 |
| .endif |
| sub r7, r7, #32 // luma_stride -= 32 |
| |
| bx r12 |
| endfunc |
| .endm |
| |
| fguv 420, 1, 1 |
| fguv 422, 1, 0 |
| fguv 444, 0, 0 |
| |
| function fguv_loop_sx0_neon |
| L(fguv_loop_sx0_tbl): |
| .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB |
| |
| .macro fguv_loop_sx0 csfl, ox, oy |
| L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): |
| sub r2, r2, #32 // src_stride -= 32 |
| sub r10, r10, #32 // grain_stride -= 32 |
| .if \oy |
| mov r12, lr |
| .endif |
| L(fguv_loop_sx0_csfl\csfl\()_\ox\oy\()_loopstart): |
| 1: |
| .if \ox |
| vld1.16 {d0}, [r4], r10 // grain_lut old |
| .endif |
| .if \oy |
| vld1.16 {q2, q3}, [r8]! // grain_lut top |
| .endif |
| .if \ox && \oy |
| vld1.16 {d2}, [r11], r10 // grain_lut top old |
| .endif |
| .if !\ox && !\oy |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| .endif |
| vld1.16 {q8, q9}, [r5]! // grain_lut |
| .if \oy |
| vld1.16 {q4, q5}, [r8], r10 // grain_lut top |
| .endif |
| .if !\ox && !\oy |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| .endif |
| .if \oy |
| vdup.16 d28, d31[2] // overlap y coeff |
| vdup.16 d29, d31[3] // overlap y coeff |
| .endif |
| vld1.16 {q10, q11}, [r5], r10 // grain_lut |
| |
| .if \ox |
| vdup.16 q7, d30[3] // bitdepth_max |
| add r4, r4, #32 |
| vmull.s16 q0, d0, d24 |
| vshr.u16 q7, q7, #1 // grain_max |
| vmlal.s16 q0, d16, d25 |
| vmvn q6, q7 // grain_min |
| .endif |
| |
| .if \oy |
| .if \ox |
| add r11, r11, #32 |
| vmull.s16 q1, d2, d24 |
| vmlal.s16 q1, d4, d25 |
| vqrshrn.s32 d16, q0, #5 |
| vqrshrn.s32 d4, q1, #5 |
| vmin.s16 d4, d4, d14 |
| vmin.s16 d16, d16, d14 |
| vmax.s16 d4, d4, d12 |
| vmax.s16 d16, d16, d12 |
| .endif |
| |
| vmull.s16 q0, d4, d28 |
| vmull.s16 q1, d5, d28 |
| vmull.s16 q2, d6, d28 |
| vmull.s16 q3, d7, d28 |
| .if !\ox |
| vdup.16 q7, d30[3] // bitdepth_max |
| .endif |
| vmlal.s16 q0, d16, d29 |
| vmlal.s16 q1, d17, d29 |
| vmlal.s16 q2, d18, d29 |
| vmlal.s16 q3, d19, d29 |
| .if !\ox |
| vshr.u16 q7, q7, #1 // grain_max |
| .endif |
| vmull.s16 q8, d20, d29 |
| vmull.s16 q9, d21, d29 |
| vmull.s16 q10, d22, d29 |
| vmull.s16 q11, d23, d29 |
| .if !\ox |
| vmvn q6, q7 // grain_min |
| .endif |
| vmlal.s16 q8, d8, d28 |
| vmlal.s16 q9, d9, d28 |
| vmlal.s16 q10, d10, d28 |
| vmlal.s16 q11, d11, d28 |
| vqrshrn.s32 d0, q0, #5 |
| vqrshrn.s32 d1, q1, #5 |
| vqrshrn.s32 d2, q2, #5 |
| vqrshrn.s32 d3, q3, #5 |
| vqrshrn.s32 d4, q8, #5 |
| vqrshrn.s32 d5, q9, #5 |
| vqrshrn.s32 d6, q10, #5 |
| vqrshrn.s32 d7, q11, #5 |
| vmin.s16 q8, q0, q7 |
| vmin.s16 q9, q1, q7 |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| vmin.s16 q10, q2, q7 |
| vmin.s16 q11, q3, q7 |
| vmax.s16 q8, q8, q6 |
| vmax.s16 q9, q9, q6 |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| vmax.s16 q10, q10, q6 |
| vmax.s16 q11, q11, q6 |
| .elseif \ox |
| vqrshrn.s32 d16, q0, #5 |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| vmin.s16 d16, d16, d14 |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| vmax.s16 d16, d16, d12 |
| .endif |
| |
| .if !\csfl |
| vdup.16 d28, d30[0] // uv_luma_mult |
| vld1.16 {q4, q5}, [r1, :128]! // src |
| vdup.16 d29, d30[1] // uv_mult |
| vmull.s16 q6, d0, d28 |
| vmull.s16 q7, d1, d28 |
| vmull.s16 q0, d2, d28 |
| vmull.s16 q1, d3, d28 |
| vmlal.s16 q6, d8, d29 |
| vmlal.s16 q7, d9, d29 |
| vmlal.s16 q0, d10, d29 |
| vmlal.s16 q1, d11, d29 |
| vld1.16 {q4, q5}, [r1, :128] // src |
| sub r1, r1, #32 |
| vshrn.s32 d12, q6, #6 |
| vshrn.s32 d13, q7, #6 |
| vshrn.s32 d14, q0, #6 |
| vshrn.s32 d15, q1, #6 |
| vmull.s16 q0, d4, d28 |
| vmull.s16 q1, d5, d28 |
| vmull.s16 q2, d6, d28 |
| vmull.s16 q3, d7, d28 |
| vmlal.s16 q0, d8, d29 |
| vmlal.s16 q1, d9, d29 |
| vmlal.s16 q2, d10, d29 |
| vmlal.s16 q3, d11, d29 |
| vdup.16 q14, d30[2] // uv_offset |
| vshrn.s32 d0, q0, #6 |
| vshrn.s32 d1, q1, #6 |
| vshrn.s32 d2, q2, #6 |
| vshrn.s32 d3, q3, #6 |
| vdup.16 q4, d30[3] // bitdepth_max |
| vmov.i16 q5, #0 |
| vadd.i16 q6, q6, q14 |
| vadd.i16 q7, q7, q14 |
| vadd.i16 q2, q0, q14 |
| vadd.i16 q3, q1, q14 |
| vmin.s16 q0, q6, q4 |
| vmin.s16 q1, q7, q4 |
| vmin.s16 q2, q2, q4 |
| vmin.s16 q3, q3, q4 |
| vmax.s16 q0, q0, q5 |
| vmax.s16 q1, q1, q5 |
| vmax.s16 q2, q2, q5 |
| vmax.s16 q3, q3, q5 |
| .else |
| vdup.16 q14, d30[3] // bitdepth_max |
| // Make sure that uninitialized pixels out of range past the right |
| // edge are in range; their actual values shouldn't matter. |
| vand q0, q0, q14 |
| vand q1, q1, q14 |
| vand q2, q2, q14 |
| vand q3, q3, q14 |
| .endif |
| |
| bl gather32_neon |
| |
| vld1.16 {q0, q1}, [r1, :128]! // src |
| |
| vmovl.u8 q6, d8 // scaling |
| vmovl.u8 q7, d9 |
| vmovl.u8 q4, d10 |
| vmovl.u8 q5, d11 |
| |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| |
| vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) |
| vshl.u16 q7, q7, q13 |
| vshl.u16 q4, q4, q13 |
| vshl.u16 q5, q5, q13 |
| |
| vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) |
| vqrdmulh.s16 q9, q9, q7 |
| vqrdmulh.s16 q10, q10, q4 |
| vqrdmulh.s16 q11, q11, q5 |
| |
| |
| vdup.16 q4, d31[0] // clip_min |
| vdup.16 q5, d31[1] // clip_max |
| |
| vqadd.s16 q0, q0, q8 // *src + noise |
| vqadd.s16 q1, q1, q9 |
| vqadd.s16 q2, q2, q10 |
| vqadd.s16 q3, q3, q11 |
| |
| .if \oy |
| vmov.32 lr, d25[0] // 2 first 16 bit coeffs from overlap x |
| .endif |
| |
| vmax.s16 q0, q0, q4 |
| vmax.s16 q1, q1, q4 |
| vmax.s16 q2, q2, q4 |
| vmax.s16 q3, q3, q4 |
| vmin.s16 q0, q0, q5 |
| vmin.s16 q1, q1, q5 |
| vmin.s16 q2, q2, q5 |
| vmin.s16 q3, q3, q5 |
| |
| vst1.16 {q0, q1}, [r0, :128]! // dst |
| |
| subs r9, r9, #1 |
| .if \oy |
| vmov.32 d31[1], lr // new coeffs for overlap y |
| .endif |
| |
| vst1.16 {q2, q3}, [r0, :128], r2 // dst |
| bgt 1b |
| |
| .if \oy |
| cmp r12, #0 |
| mov r9, r12 // restore actual remaining h |
| bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0_loopstart) |
| .endif |
| b 9f |
| .endm |
| fguv_loop_sx0 0, 0, 0 |
| fguv_loop_sx0 0, 0, 1 |
| fguv_loop_sx0 0, 1, 0 |
| fguv_loop_sx0 0, 1, 1 |
| fguv_loop_sx0 1, 0, 0 |
| fguv_loop_sx0 1, 0, 1 |
| fguv_loop_sx0 1, 1, 0 |
| fguv_loop_sx0 1, 1, 1 |
| |
| 9: |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| endfunc |
| |
| function fguv_loop_sx1_neon |
| L(fguv_loop_sx1_tbl): |
| .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB |
| |
| .macro fguv_loop_sx1 csfl, ox, oy |
| L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): |
| .if \oy |
| mov r12, lr |
| .endif |
| 1: |
| .if \ox |
| vld1.16 {d0}, [r4], r10 // grain_lut old |
| .endif |
| .if \ox && \oy |
| vld1.16 {d2}, [r11], r10 // grain_lut top old |
| .endif |
| .if \oy |
| vld1.16 {q2, q3}, [r8], r10 // grain_lut top |
| .endif |
| .if !\ox && !\oy |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| .endif |
| vld1.16 {q8, q9}, [r5], r10 // grain_lut |
| .if \oy |
| vdup.16 d28, d31[2] // overlap y coeff |
| vdup.16 d29, d31[3] // overlap y coeff |
| .endif |
| .if !\ox && !\oy |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| .endif |
| |
| .if \ox |
| vdup.16 q7, d30[3] // bitdepth_max |
| vmull.s16 q0, d0, d24 |
| vshr.u16 q7, q7, #1 // grain_max |
| vmlal.s16 q0, d16, d25 |
| vmvn q6, q7 // grain_min |
| .endif |
| |
| .if \oy |
| .if \ox |
| vmull.s16 q1, d2, d24 |
| vmlal.s16 q1, d4, d25 |
| vqrshrn.s32 d16, q0, #5 |
| vqrshrn.s32 d4, q1, #5 |
| vmin.s16 d4, d4, d14 |
| vmin.s16 d16, d16, d14 |
| vmax.s16 d4, d4, d12 |
| vmax.s16 d16, d16, d12 |
| .endif |
| |
| vmull.s16 q0, d4, d28 |
| vmull.s16 q1, d5, d28 |
| vmull.s16 q2, d6, d28 |
| vmull.s16 q3, d7, d28 |
| .if !\ox |
| vdup.16 q7, d30[3] // bitdepth_max |
| .endif |
| vmlal.s16 q0, d16, d29 |
| vmlal.s16 q1, d17, d29 |
| vmlal.s16 q2, d18, d29 |
| vmlal.s16 q3, d19, d29 |
| .if !\ox |
| vshr.u16 q7, q7, #1 // grain_max |
| .endif |
| vqrshrn.s32 d16, q0, #5 |
| vqrshrn.s32 d17, q1, #5 |
| vqrshrn.s32 d18, q2, #5 |
| vqrshrn.s32 d19, q3, #5 |
| .if !\ox |
| vmvn q6, q7 // grain_min |
| .endif |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| vmin.s16 q8, q8, q7 |
| vmin.s16 q9, q9, q7 |
| vmax.s16 q8, q8, q6 |
| vmax.s16 q9, q9, q6 |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| .elseif \ox |
| vqrshrn.s32 d16, q0, #5 |
| vld1.16 {q0, q1}, [r6, :128]! // luma |
| vmin.s16 d16, d16, d14 |
| vld1.16 {q2, q3}, [r6, :128], r7 // luma |
| vmax.s16 d16, d16, d12 |
| .endif |
| |
| vpadd.i16 d0, d0, d1 |
| vpadd.i16 d1, d2, d3 |
| vpadd.i16 d2, d4, d5 |
| vpadd.i16 d3, d6, d7 |
| vrshr.u16 q0, q0, #1 |
| vrshr.u16 q1, q1, #1 |
| .if !\csfl |
| vdup.16 d28, d30[0] // uv_luma_mult |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| vdup.16 d29, d30[1] // uv_mult |
| vmull.s16 q6, d0, d28 |
| vmull.s16 q7, d1, d28 |
| vmull.s16 q0, d2, d28 |
| vmull.s16 q1, d3, d28 |
| vmlal.s16 q6, d4, d29 |
| vmlal.s16 q7, d5, d29 |
| vmlal.s16 q0, d6, d29 |
| vmlal.s16 q1, d7, d29 |
| vshrn.s32 d12, q6, #6 |
| vshrn.s32 d13, q7, #6 |
| vshrn.s32 d14, q0, #6 |
| vshrn.s32 d15, q1, #6 |
| vdup.16 q14, d30[2] // uv_offset |
| vdup.16 q4, d30[3] // bitdepth_max |
| vmov.i16 q5, #0 |
| vadd.i16 q6, q6, q14 |
| vadd.i16 q7, q7, q14 |
| vmin.s16 q0, q6, q4 |
| vmin.s16 q1, q7, q4 |
| vmax.s16 q0, q0, q5 |
| vmax.s16 q1, q1, q5 |
| .else |
| vdup.16 q14, d30[3] // bitdepth_max |
| vld1.16 {q2, q3}, [r1, :128], r2 // src |
| |
| // Make sure that uninitialized pixels out of range past the right |
| // edge are in range; their actual values shouldn't matter. |
| vand q0, q0, q14 |
| vand q1, q1, q14 |
| .endif |
| |
| bl gather16_neon |
| |
| vmovl.u8 q6, d8 // scaling |
| vmovl.u8 q7, d9 |
| |
| vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) |
| vshl.u16 q7, q7, q13 |
| |
| vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) |
| vqrdmulh.s16 q9, q9, q7 |
| |
| |
| vdup.16 q4, d31[0] // clip_min |
| vdup.16 q5, d31[1] // clip_max |
| |
| vqadd.s16 q0, q2, q8 // *src + noise |
| vqadd.s16 q1, q3, q9 |
| |
| .if \oy |
| // Swap the two last coefficients of d31, place them first in d28 |
| vrev64.16 d28, d31 |
| .endif |
| |
| vmax.s16 q0, q0, q4 |
| vmax.s16 q1, q1, q4 |
| vmin.s16 q0, q0, q5 |
| vmin.s16 q1, q1, q5 |
| |
| subs r9, r9, #1 |
| .if \oy |
| // Take the first two 16 bit coefficients of d28 and place them at the |
| // end of d31 |
| vtrn.32 d31, d28 |
| .endif |
| |
| vst1.16 {q0, q1}, [r0, :128], r2 // dst |
| bgt 1b |
| |
| .if \oy |
| cmp r12, #0 |
| mov r9, r12 // restore actual remaining h |
| bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) |
| .endif |
| |
| b 9f |
| .endm |
| fguv_loop_sx1 0, 0, 0 |
| fguv_loop_sx1 0, 0, 1 |
| fguv_loop_sx1 0, 1, 0 |
| fguv_loop_sx1 0, 1, 1 |
| fguv_loop_sx1 1, 0, 0 |
| fguv_loop_sx1 1, 0, 1 |
| fguv_loop_sx1 1, 1, 0 |
| fguv_loop_sx1 1, 1, 1 |
| |
| 9: |
| vpop {q4-q7} |
| pop {r4-r11,pc} |
| endfunc |
