| /*********************************************************************** |
| Copyright (c) 2017 Google Inc. |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions |
| are met: |
| - Redistributions of source code must retain the above copyright notice, |
| this list of conditions and the following disclaimer. |
| - Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| - Neither the name of Internet Society, IETF or IETF Trust, nor the |
| names of specific contributors, may be used to endorse or promote |
| products derived from this software without specific prior written |
| permission. |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| POSSIBILITY OF SUCH DAMAGE. |
| ***********************************************************************/ |
| |
| #ifdef HAVE_CONFIG_H |
| #include "config.h" |
| #endif |
| |
| #include <arm_neon.h> |
| #include "SigProc_FIX.h" |
| #include "define.h" |
| |
| #define QA 24 |
| #define A_LIMIT SILK_FIX_CONST( 0.99975, QA ) |
| |
| #define MUL32_FRAC_Q(a32, b32, Q) ((opus_int32)(silk_RSHIFT_ROUND64(silk_SMULL(a32, b32), Q))) |
| |
| /* The difficulty is how to judge a 64-bit signed integer tmp64 is 32-bit overflowed, |
| * since NEON has no 64-bit min, max or comparison instructions. |
| * A failed idea is to compare the results of vmovn(tmp64) and vqmovn(tmp64) whether they are equal or not. |
| * However, this idea fails when the tmp64 is something like 0xFFFFFFF980000000. |
| * Here we know that mult2Q >= 1, so the highest bit (bit 63, sign bit) of tmp64 must equal to bit 62. |
| * tmp64 was shifted left by 1 and we got tmp64'. If high_half(tmp64') != 0 and high_half(tmp64') != -1, |
| * then we know that bit 31 to bit 63 of tmp64 can not all be the sign bit, and therefore tmp64 is 32-bit overflowed. |
| * That is, we judge if tmp64' > 0x00000000FFFFFFFF, or tmp64' <= 0xFFFFFFFF00000000. |
| * We use narrowing shift right 31 bits to tmp32' to save data bandwidth and instructions. |
| * That is, we judge if tmp32' > 0x00000000, or tmp32' <= 0xFFFFFFFF. |
| */ |
| |
| /* Compute inverse of LPC prediction gain, and */ |
| /* test if LPC coefficients are stable (all poles within unit circle) */ |
| static OPUS_INLINE opus_int32 LPC_inverse_pred_gain_QA_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ |
| opus_int32 A_QA[ SILK_MAX_ORDER_LPC ], /* I Prediction coefficients */ |
| const opus_int order /* I Prediction order */ |
| ) |
| { |
| opus_int k, n, mult2Q; |
| opus_int32 invGain_Q30, rc_Q31, rc_mult1_Q30, rc_mult2, tmp1, tmp2; |
| opus_int32 max, min; |
| int32x4_t max_s32x4, min_s32x4; |
| int32x2_t max_s32x2, min_s32x2; |
| |
| max_s32x4 = vdupq_n_s32( silk_int32_MIN ); |
| min_s32x4 = vdupq_n_s32( silk_int32_MAX ); |
| invGain_Q30 = SILK_FIX_CONST( 1, 30 ); |
| for( k = order - 1; k > 0; k-- ) { |
| int32x2_t rc_Q31_s32x2, rc_mult2_s32x2; |
| int64x2_t mult2Q_s64x2; |
| |
| /* Check for stability */ |
| if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { |
| return 0; |
| } |
| |
| /* Set RC equal to negated AR coef */ |
| rc_Q31 = -silk_LSHIFT( A_QA[ k ], 31 - QA ); |
| |
| /* rc_mult1_Q30 range: [ 1 : 2^30 ] */ |
| rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); |
| silk_assert( rc_mult1_Q30 > ( 1 << 15 ) ); /* reduce A_LIMIT if fails */ |
| silk_assert( rc_mult1_Q30 <= ( 1 << 30 ) ); |
| |
| /* Update inverse gain */ |
| /* invGain_Q30 range: [ 0 : 2^30 ] */ |
| invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); |
| silk_assert( invGain_Q30 >= 0 ); |
| silk_assert( invGain_Q30 <= ( 1 << 30 ) ); |
| if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { |
| return 0; |
| } |
| |
| /* rc_mult2 range: [ 2^30 : silk_int32_MAX ] */ |
| mult2Q = 32 - silk_CLZ32( silk_abs( rc_mult1_Q30 ) ); |
| rc_mult2 = silk_INVERSE32_varQ( rc_mult1_Q30, mult2Q + 30 ); |
| |
| /* Update AR coefficient */ |
| rc_Q31_s32x2 = vdup_n_s32( rc_Q31 ); |
| mult2Q_s64x2 = vdupq_n_s64( -mult2Q ); |
| rc_mult2_s32x2 = vdup_n_s32( rc_mult2 ); |
| |
| for( n = 0; n < ( ( k + 1 ) >> 1 ) - 3; n += 4 ) { |
| /* We always calculate extra elements of A_QA buffer when ( k % 4 ) != 0, to take the advantage of SIMD parallelization. */ |
| int32x4_t tmp1_s32x4, tmp2_s32x4, t0_s32x4, t1_s32x4, s0_s32x4, s1_s32x4, t_QA0_s32x4, t_QA1_s32x4; |
| int64x2_t t0_s64x2, t1_s64x2, t2_s64x2, t3_s64x2; |
| tmp1_s32x4 = vld1q_s32( A_QA + n ); |
| tmp2_s32x4 = vld1q_s32( A_QA + k - n - 4 ); |
| tmp2_s32x4 = vrev64q_s32( tmp2_s32x4 ); |
| tmp2_s32x4 = vcombine_s32( vget_high_s32( tmp2_s32x4 ), vget_low_s32( tmp2_s32x4 ) ); |
| t0_s32x4 = vqrdmulhq_lane_s32( tmp2_s32x4, rc_Q31_s32x2, 0 ); |
| t1_s32x4 = vqrdmulhq_lane_s32( tmp1_s32x4, rc_Q31_s32x2, 0 ); |
| t_QA0_s32x4 = vqsubq_s32( tmp1_s32x4, t0_s32x4 ); |
| t_QA1_s32x4 = vqsubq_s32( tmp2_s32x4, t1_s32x4 ); |
| t0_s64x2 = vmull_s32( vget_low_s32 ( t_QA0_s32x4 ), rc_mult2_s32x2 ); |
| t1_s64x2 = vmull_s32( vget_high_s32( t_QA0_s32x4 ), rc_mult2_s32x2 ); |
| t2_s64x2 = vmull_s32( vget_low_s32 ( t_QA1_s32x4 ), rc_mult2_s32x2 ); |
| t3_s64x2 = vmull_s32( vget_high_s32( t_QA1_s32x4 ), rc_mult2_s32x2 ); |
| t0_s64x2 = vrshlq_s64( t0_s64x2, mult2Q_s64x2 ); |
| t1_s64x2 = vrshlq_s64( t1_s64x2, mult2Q_s64x2 ); |
| t2_s64x2 = vrshlq_s64( t2_s64x2, mult2Q_s64x2 ); |
| t3_s64x2 = vrshlq_s64( t3_s64x2, mult2Q_s64x2 ); |
| t0_s32x4 = vcombine_s32( vmovn_s64( t0_s64x2 ), vmovn_s64( t1_s64x2 ) ); |
| t1_s32x4 = vcombine_s32( vmovn_s64( t2_s64x2 ), vmovn_s64( t3_s64x2 ) ); |
| s0_s32x4 = vcombine_s32( vshrn_n_s64( t0_s64x2, 31 ), vshrn_n_s64( t1_s64x2, 31 ) ); |
| s1_s32x4 = vcombine_s32( vshrn_n_s64( t2_s64x2, 31 ), vshrn_n_s64( t3_s64x2, 31 ) ); |
| max_s32x4 = vmaxq_s32( max_s32x4, s0_s32x4 ); |
| min_s32x4 = vminq_s32( min_s32x4, s0_s32x4 ); |
| max_s32x4 = vmaxq_s32( max_s32x4, s1_s32x4 ); |
| min_s32x4 = vminq_s32( min_s32x4, s1_s32x4 ); |
| t1_s32x4 = vrev64q_s32( t1_s32x4 ); |
| t1_s32x4 = vcombine_s32( vget_high_s32( t1_s32x4 ), vget_low_s32( t1_s32x4 ) ); |
| vst1q_s32( A_QA + n, t0_s32x4 ); |
| vst1q_s32( A_QA + k - n - 4, t1_s32x4 ); |
| } |
| for( ; n < (k + 1) >> 1; n++ ) { |
| opus_int64 tmp64; |
| tmp1 = A_QA[ n ]; |
| tmp2 = A_QA[ k - n - 1 ]; |
| tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp1, |
| MUL32_FRAC_Q( tmp2, rc_Q31, 31 ) ), rc_mult2 ), mult2Q); |
| if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { |
| return 0; |
| } |
| A_QA[ n ] = ( opus_int32 )tmp64; |
| tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp2, |
| MUL32_FRAC_Q( tmp1, rc_Q31, 31 ) ), rc_mult2), mult2Q); |
| if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { |
| return 0; |
| } |
| A_QA[ k - n - 1 ] = ( opus_int32 )tmp64; |
| } |
| } |
| |
| /* Check for stability */ |
| if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { |
| return 0; |
| } |
| |
| max_s32x2 = vmax_s32( vget_low_s32( max_s32x4 ), vget_high_s32( max_s32x4 ) ); |
| min_s32x2 = vmin_s32( vget_low_s32( min_s32x4 ), vget_high_s32( min_s32x4 ) ); |
| max_s32x2 = vmax_s32( max_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( max_s32x2 ), 32 ) ) ); |
| min_s32x2 = vmin_s32( min_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( min_s32x2 ), 32 ) ) ); |
| max = vget_lane_s32( max_s32x2, 0 ); |
| min = vget_lane_s32( min_s32x2, 0 ); |
| if( ( max > 0 ) || ( min < -1 ) ) { |
| return 0; |
| } |
| |
| /* Set RC equal to negated AR coef */ |
| rc_Q31 = -silk_LSHIFT( A_QA[ 0 ], 31 - QA ); |
| |
| /* Range: [ 1 : 2^30 ] */ |
| rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); |
| |
| /* Update inverse gain */ |
| /* Range: [ 0 : 2^30 ] */ |
| invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); |
| silk_assert( invGain_Q30 >= 0 ); |
| silk_assert( invGain_Q30 <= ( 1 << 30 ) ); |
| if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { |
| return 0; |
| } |
| |
| return invGain_Q30; |
| } |
| |
| /* For input in Q12 domain */ |
| opus_int32 silk_LPC_inverse_pred_gain_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ |
| const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ |
| const opus_int order /* I Prediction order */ |
| ) |
| { |
| #ifdef OPUS_CHECK_ASM |
| const opus_int32 invGain_Q30_c = silk_LPC_inverse_pred_gain_c( A_Q12, order ); |
| #endif |
| |
| opus_int32 invGain_Q30; |
| if( ( SILK_MAX_ORDER_LPC != 24 ) || ( order & 1 )) { |
| invGain_Q30 = silk_LPC_inverse_pred_gain_c( A_Q12, order ); |
| } |
| else { |
| opus_int32 Atmp_QA[ SILK_MAX_ORDER_LPC ]; |
| opus_int32 DC_resp; |
| int16x8_t t0_s16x8, t1_s16x8, t2_s16x8; |
| int32x4_t t0_s32x4; |
| const opus_int leftover = order & 7; |
| |
| /* Increase Q domain of the AR coefficients */ |
| t0_s16x8 = vld1q_s16( A_Q12 + 0 ); |
| t1_s16x8 = vld1q_s16( A_Q12 + 8 ); |
| t2_s16x8 = vld1q_s16( A_Q12 + 16 ); |
| t0_s32x4 = vpaddlq_s16( t0_s16x8 ); |
| |
| switch( order - leftover ) |
| { |
| case 24: |
| t0_s32x4 = vpadalq_s16( t0_s32x4, t2_s16x8 ); |
| /* Intend to fall through */ |
| |
| case 16: |
| t0_s32x4 = vpadalq_s16( t0_s32x4, t1_s16x8 ); |
| vst1q_s32( Atmp_QA + 16, vshll_n_s16( vget_low_s16 ( t2_s16x8 ), QA - 12 ) ); |
| vst1q_s32( Atmp_QA + 20, vshll_n_s16( vget_high_s16( t2_s16x8 ), QA - 12 ) ); |
| /* Intend to fall through */ |
| |
| case 8: |
| { |
| const int32x2_t t_s32x2 = vpadd_s32( vget_low_s32( t0_s32x4 ), vget_high_s32( t0_s32x4 ) ); |
| const int64x1_t t_s64x1 = vpaddl_s32( t_s32x2 ); |
| DC_resp = vget_lane_s32( vreinterpret_s32_s64( t_s64x1 ), 0 ); |
| vst1q_s32( Atmp_QA + 8, vshll_n_s16( vget_low_s16 ( t1_s16x8 ), QA - 12 ) ); |
| vst1q_s32( Atmp_QA + 12, vshll_n_s16( vget_high_s16( t1_s16x8 ), QA - 12 ) ); |
| } |
| break; |
| |
| default: |
| DC_resp = 0; |
| break; |
| } |
| A_Q12 += order - leftover; |
| |
| switch( leftover ) |
| { |
| case 6: |
| DC_resp += (opus_int32)A_Q12[ 5 ]; |
| DC_resp += (opus_int32)A_Q12[ 4 ]; |
| /* Intend to fall through */ |
| |
| case 4: |
| DC_resp += (opus_int32)A_Q12[ 3 ]; |
| DC_resp += (opus_int32)A_Q12[ 2 ]; |
| /* Intend to fall through */ |
| |
| case 2: |
| DC_resp += (opus_int32)A_Q12[ 1 ]; |
| DC_resp += (opus_int32)A_Q12[ 0 ]; |
| /* Intend to fall through */ |
| |
| default: |
| break; |
| } |
| |
| /* If the DC is unstable, we don't even need to do the full calculations */ |
| if( DC_resp >= 4096 ) { |
| invGain_Q30 = 0; |
| } else { |
| vst1q_s32( Atmp_QA + 0, vshll_n_s16( vget_low_s16 ( t0_s16x8 ), QA - 12 ) ); |
| vst1q_s32( Atmp_QA + 4, vshll_n_s16( vget_high_s16( t0_s16x8 ), QA - 12 ) ); |
| invGain_Q30 = LPC_inverse_pred_gain_QA_neon( Atmp_QA, order ); |
| } |
| } |
| |
| #ifdef OPUS_CHECK_ASM |
| silk_assert( invGain_Q30_c == invGain_Q30 ); |
| #endif |
| |
| return invGain_Q30; |
| } |
