src/third_party/libvpx/vpx_dsp/x86/convolve_avx2.h

/*
 *  Copyright (c) 2017 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef VPX_VPX_DSP_X86_CONVOLVE_AVX2_H_
#define VPX_VPX_DSP_X86_CONVOLVE_AVX2_H_

#include <immintrin.h>  // AVX2

#include "./vpx_config.h"

#if defined(__clang__)
#if (__clang_major__ > 0 && __clang_major__ < 3) ||            \
    (__clang_major__ == 3 && __clang_minor__ <= 3) ||          \
    (defined(__APPLE__) && defined(__apple_build_version__) && \
     ((__clang_major__ == 4 && __clang_minor__ <= 2) ||        \
      (__clang_major__ == 5 && __clang_minor__ == 0)))
#define MM256_BROADCASTSI128_SI256(x) \
  _mm_broadcastsi128_si256((__m128i const *)&(x))
#else  // clang > 3.3, and not 5.0 on macosx.
#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x)
#endif  // clang <= 3.3
#elif defined(__GNUC__)
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ <= 6)
#define MM256_BROADCASTSI128_SI256(x) \
  _mm_broadcastsi128_si256((__m128i const *)&(x))
#elif __GNUC__ == 4 && __GNUC_MINOR__ == 7
#define MM256_BROADCASTSI128_SI256(x) _mm_broadcastsi128_si256(x)
#else  // gcc > 4.7
#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x)
#endif  // gcc <= 4.6
#else   // !(gcc || clang)
#define MM256_BROADCASTSI128_SI256(x) _mm256_broadcastsi128_si256(x)
#endif  // __clang__

static INLINE void shuffle_filter_avx2(const int16_t *const filter,
                                       __m256i *const f) {
  const __m256i f_values =
      MM256_BROADCASTSI128_SI256(_mm_load_si128((const __m128i *)filter));
  // pack and duplicate the filter values
  f[0] = _mm256_shuffle_epi8(f_values, _mm256_set1_epi16(0x0200u));
  f[1] = _mm256_shuffle_epi8(f_values, _mm256_set1_epi16(0x0604u));
  f[2] = _mm256_shuffle_epi8(f_values, _mm256_set1_epi16(0x0a08u));
  f[3] = _mm256_shuffle_epi8(f_values, _mm256_set1_epi16(0x0e0cu));
}

static INLINE __m256i convolve8_16_avx2(const __m256i *const s,
                                        const __m256i *const f) {
  // multiply 2 adjacent elements with the filter and add the result
  const __m256i k_64 = _mm256_set1_epi16(1 << 6);
  const __m256i x0 = _mm256_maddubs_epi16(s[0], f[0]);
  const __m256i x1 = _mm256_maddubs_epi16(s[1], f[1]);
  const __m256i x2 = _mm256_maddubs_epi16(s[2], f[2]);
  const __m256i x3 = _mm256_maddubs_epi16(s[3], f[3]);
  __m256i sum1, sum2;

  // sum the results together, saturating only on the final step
  // adding x0 with x2 and x1 with x3 is the only order that prevents
  // outranges for all filters
  sum1 = _mm256_add_epi16(x0, x2);
  sum2 = _mm256_add_epi16(x1, x3);
  // add the rounding offset early to avoid another saturated add
  sum1 = _mm256_add_epi16(sum1, k_64);
  sum1 = _mm256_adds_epi16(sum1, sum2);
  // round and shift by 7 bit each 16 bit
  sum1 = _mm256_srai_epi16(sum1, 7);
  return sum1;
}

static INLINE __m128i convolve8_8_avx2(const __m256i *const s,
                                       const __m256i *const f) {
  // multiply 2 adjacent elements with the filter and add the result
  const __m128i k_64 = _mm_set1_epi16(1 << 6);
  const __m128i x0 = _mm_maddubs_epi16(_mm256_castsi256_si128(s[0]),
                                       _mm256_castsi256_si128(f[0]));
  const __m128i x1 = _mm_maddubs_epi16(_mm256_castsi256_si128(s[1]),
                                       _mm256_castsi256_si128(f[1]));
  const __m128i x2 = _mm_maddubs_epi16(_mm256_castsi256_si128(s[2]),
                                       _mm256_castsi256_si128(f[2]));
  const __m128i x3 = _mm_maddubs_epi16(_mm256_castsi256_si128(s[3]),
                                       _mm256_castsi256_si128(f[3]));
  __m128i sum1, sum2;

  // sum the results together, saturating only on the final step
  // adding x0 with x2 and x1 with x3 is the only order that prevents
  // outranges for all filters
  sum1 = _mm_add_epi16(x0, x2);
  sum2 = _mm_add_epi16(x1, x3);
  // add the rounding offset early to avoid another saturated add
  sum1 = _mm_add_epi16(sum1, k_64);
  sum1 = _mm_adds_epi16(sum1, sum2);
  // shift by 7 bit each 16 bit
  sum1 = _mm_srai_epi16(sum1, 7);
  return sum1;
}

static INLINE __m256i mm256_loadu2_si128(const void *lo, const void *hi) {
  const __m256i tmp =
      _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)lo));
  return _mm256_inserti128_si256(tmp, _mm_loadu_si128((const __m128i *)hi), 1);
}

static INLINE __m256i mm256_loadu2_epi64(const void *lo, const void *hi) {
  const __m256i tmp =
      _mm256_castsi128_si256(_mm_loadl_epi64((const __m128i *)lo));
  return _mm256_inserti128_si256(tmp, _mm_loadl_epi64((const __m128i *)hi), 1);
}

static INLINE void mm256_store2_si128(__m128i *const dst_ptr_1,
                                      __m128i *const dst_ptr_2,
                                      const __m256i *const src) {
  _mm_store_si128(dst_ptr_1, _mm256_castsi256_si128(*src));
  _mm_store_si128(dst_ptr_2, _mm256_extractf128_si256(*src, 1));
}

static INLINE void mm256_storeu2_epi64(__m128i *const dst_ptr_1,
                                       __m128i *const dst_ptr_2,
                                       const __m256i *const src) {
  _mm_storel_epi64(dst_ptr_1, _mm256_castsi256_si128(*src));
  _mm_storel_epi64(dst_ptr_2, _mm256_extractf128_si256(*src, 1));
}

static INLINE void mm256_storeu2_epi32(__m128i *const dst_ptr_1,
                                       __m128i *const dst_ptr_2,
                                       const __m256i *const src) {
  *((uint32_t *)(dst_ptr_1)) = _mm_cvtsi128_si32(_mm256_castsi256_si128(*src));
  *((uint32_t *)(dst_ptr_2)) =
      _mm_cvtsi128_si32(_mm256_extractf128_si256(*src, 1));
}

static INLINE __m256i mm256_round_epi32(const __m256i *const src,
                                        const __m256i *const half_depth,
                                        const int depth) {
  const __m256i nearest_src = _mm256_add_epi32(*src, *half_depth);
  return _mm256_srai_epi32(nearest_src, depth);
}

static INLINE __m256i mm256_round_epi16(const __m256i *const src,
                                        const __m256i *const half_depth,
                                        const int depth) {
  const __m256i nearest_src = _mm256_adds_epi16(*src, *half_depth);
  return _mm256_srai_epi16(nearest_src, depth);
}

static INLINE __m256i mm256_madd_add_epi32(const __m256i *const src_0,
                                           const __m256i *const src_1,
                                           const __m256i *const ker_0,
                                           const __m256i *const ker_1) {
  const __m256i tmp_0 = _mm256_madd_epi16(*src_0, *ker_0);
  const __m256i tmp_1 = _mm256_madd_epi16(*src_1, *ker_1);
  return _mm256_add_epi32(tmp_0, tmp_1);
}

#undef MM256_BROADCASTSI128_SI256

#endif  // VPX_VPX_DSP_X86_CONVOLVE_AVX2_H_