volk_16i_32fc_dot_prod_32fc.h

Go to the documentation of this file.

/* -*- c++ -*- */
/*
 * Copyright 2012, 2014 Free Software Foundation, Inc.
 *
 * This file is part of VOLK
 *
 * SPDX-License-Identifier: LGPL-3.0-or-later
 */

 
#ifndef INCLUDED_volk_16i_32fc_dot_prod_32fc_H
#define INCLUDED_volk_16i_32fc_dot_prod_32fc_H
 
#include <stdio.h>
#include <volk/volk_common.h>
 
 
#ifdef LV_HAVE_GENERIC
 
static inline void volk_16i_32fc_dot_prod_32fc_generic(lv_32fc_t* result,
                                                       const short* input,
                                                       const lv_32fc_t* taps,
                                                       unsigned int num_points)
{
 
    static const int N_UNROLL = 4;
 
    lv_32fc_t acc0 = 0;
    lv_32fc_t acc1 = 0;
    lv_32fc_t acc2 = 0;
    lv_32fc_t acc3 = 0;
 
    unsigned i = 0;
    unsigned n = (num_points / N_UNROLL) * N_UNROLL;
 
    for (i = 0; i < n; i += N_UNROLL) {
        acc0 += taps[i + 0] * (float)input[i + 0];
        acc1 += taps[i + 1] * (float)input[i + 1];
        acc2 += taps[i + 2] * (float)input[i + 2];
        acc3 += taps[i + 3] * (float)input[i + 3];
    }
 
    for (; i < num_points; i++) {
        acc0 += taps[i] * (float)input[i];
    }
 
    *result = acc0 + acc1 + acc2 + acc3;
}
 
#endif /*LV_HAVE_GENERIC*/
 
#ifdef LV_HAVE_NEON
#include <arm_neon.h>
static inline void volk_16i_32fc_dot_prod_32fc_neon(lv_32fc_t* result,
                                                    const short* input,
                                                    const lv_32fc_t* taps,
                                                    unsigned int num_points)
{
 
    unsigned ii;
    unsigned quarter_points = num_points / 4;
    lv_32fc_t* tapsPtr = (lv_32fc_t*)taps;
    short* inputPtr = (short*)input;
    lv_32fc_t accumulator_vec[4];
 
    float32x4x2_t tapsVal, accumulator_val;
    int16x4_t input16;
    int32x4_t input32;
    float32x4_t input_float, prod_re, prod_im;
 
    accumulator_val.val[0] = vdupq_n_f32(0.0);
    accumulator_val.val[1] = vdupq_n_f32(0.0);
 
    for (ii = 0; ii < quarter_points; ++ii) {
        tapsVal = vld2q_f32((float*)tapsPtr);
        input16 = vld1_s16(inputPtr);
        // widen 16-bit int to 32-bit int
        input32 = vmovl_s16(input16);
        // convert 32-bit int to float with scale
        input_float = vcvtq_f32_s32(input32);
 
        prod_re = vmulq_f32(input_float, tapsVal.val[0]);
        prod_im = vmulq_f32(input_float, tapsVal.val[1]);
 
        accumulator_val.val[0] = vaddq_f32(prod_re, accumulator_val.val[0]);
        accumulator_val.val[1] = vaddq_f32(prod_im, accumulator_val.val[1]);
 
        tapsPtr += 4;
        inputPtr += 4;
    }
    vst2q_f32((float*)accumulator_vec, accumulator_val);
    accumulator_vec[0] += accumulator_vec[1];
    accumulator_vec[2] += accumulator_vec[3];
    accumulator_vec[0] += accumulator_vec[2];
 
    for (ii = quarter_points * 4; ii < num_points; ++ii) {
        accumulator_vec[0] += *(tapsPtr++) * (float)(*(inputPtr++));
    }
 
    *result = accumulator_vec[0];
}
 
#endif /*LV_HAVE_NEON*/
 
#ifdef LV_HAVE_NEONV8
#include <arm_neon.h>
 
static inline void volk_16i_32fc_dot_prod_32fc_neonv8(lv_32fc_t* result,
                                                      const short* input,
                                                      const lv_32fc_t* taps,
                                                      unsigned int num_points)
{
    const unsigned int eighthPoints = num_points / 8;
    const short* inputPtr = input;
    const lv_32fc_t* tapsPtr = taps;
 
    /* Use 2 independent real/imag accumulators for FMA pipelining */
    float32x4_t real_acc0 = vdupq_n_f32(0);
    float32x4_t imag_acc0 = vdupq_n_f32(0);
    float32x4_t real_acc1 = vdupq_n_f32(0);
    float32x4_t imag_acc1 = vdupq_n_f32(0);
 
    for (unsigned int number = 0; number < eighthPoints; number++) {
        /* Load 8 int16 values and convert to float */
        int16x8_t input16 = vld1q_s16(inputPtr);
        float32x4_t input_lo = vcvtq_f32_s32(vmovl_s16(vget_low_s16(input16)));
        float32x4_t input_hi = vcvtq_f32_s32(vmovl_s16(vget_high_s16(input16)));
 
        /* Load 8 complex taps deinterleaved */
        float32x4x2_t taps0 = vld2q_f32((const float*)tapsPtr);
        float32x4x2_t taps1 = vld2q_f32((const float*)(tapsPtr + 4));
        __VOLK_PREFETCH(inputPtr + 16);
        __VOLK_PREFETCH(tapsPtr + 16);
 
        /* FMA: acc += input * taps */
        real_acc0 = vfmaq_f32(real_acc0, input_lo, taps0.val[0]);
        imag_acc0 = vfmaq_f32(imag_acc0, input_lo, taps0.val[1]);
        real_acc1 = vfmaq_f32(real_acc1, input_hi, taps1.val[0]);
        imag_acc1 = vfmaq_f32(imag_acc1, input_hi, taps1.val[1]);
 
        inputPtr += 8;
        tapsPtr += 8;
    }
 
    /* Combine accumulators */
    real_acc0 = vaddq_f32(real_acc0, real_acc1);
    imag_acc0 = vaddq_f32(imag_acc0, imag_acc1);
 
    /* Horizontal sum */
    float real_sum = vaddvq_f32(real_acc0);
    float imag_sum = vaddvq_f32(imag_acc0);
 
    lv_32fc_t returnValue = lv_cmake(real_sum, imag_sum);
 
    /* Handle remainder */
    const float* bPtr = (const float*)tapsPtr;
    for (unsigned int number = eighthPoints * 8; number < num_points; number++) {
        returnValue += lv_cmake(inputPtr[0] * bPtr[0], inputPtr[0] * bPtr[1]);
        inputPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
#endif /*LV_HAVE_NEONV8*/
 
#if LV_HAVE_SSE && LV_HAVE_MMX
 
static inline void volk_16i_32fc_dot_prod_32fc_u_sse(lv_32fc_t* result,
                                                     const short* input,
                                                     const lv_32fc_t* taps,
                                                     unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int eighthPoints = num_points / 8;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m64 m0, m1;
    __m128 f0, f1, f2, f3;
    __m128 a0Val, a1Val, a2Val, a3Val;
    __m128 b0Val, b1Val, b2Val, b3Val;
    __m128 c0Val, c1Val, c2Val, c3Val;
 
    __m128 dotProdVal0 = _mm_setzero_ps();
    __m128 dotProdVal1 = _mm_setzero_ps();
    __m128 dotProdVal2 = _mm_setzero_ps();
    __m128 dotProdVal3 = _mm_setzero_ps();
 
    for (; number < eighthPoints; number++) {
 
        m0 = _mm_set_pi16(*(aPtr + 3), *(aPtr + 2), *(aPtr + 1), *(aPtr + 0));
        m1 = _mm_set_pi16(*(aPtr + 7), *(aPtr + 6), *(aPtr + 5), *(aPtr + 4));
        f0 = _mm_cvtpi16_ps(m0);
        f1 = _mm_cvtpi16_ps(m0);
        f2 = _mm_cvtpi16_ps(m1);
        f3 = _mm_cvtpi16_ps(m1);
 
        a0Val = _mm_unpacklo_ps(f0, f1);
        a1Val = _mm_unpackhi_ps(f0, f1);
        a2Val = _mm_unpacklo_ps(f2, f3);
        a3Val = _mm_unpackhi_ps(f2, f3);
 
        b0Val = _mm_loadu_ps(bPtr);
        b1Val = _mm_loadu_ps(bPtr + 4);
        b2Val = _mm_loadu_ps(bPtr + 8);
        b3Val = _mm_loadu_ps(bPtr + 12);
 
        c0Val = _mm_mul_ps(a0Val, b0Val);
        c1Val = _mm_mul_ps(a1Val, b1Val);
        c2Val = _mm_mul_ps(a2Val, b2Val);
        c3Val = _mm_mul_ps(a3Val, b3Val);
 
        dotProdVal0 = _mm_add_ps(c0Val, dotProdVal0);
        dotProdVal1 = _mm_add_ps(c1Val, dotProdVal1);
        dotProdVal2 = _mm_add_ps(c2Val, dotProdVal2);
        dotProdVal3 = _mm_add_ps(c3Val, dotProdVal3);
 
        aPtr += 8;
        bPtr += 16;
    }
 
    _mm_empty(); // clear the mmx technology state
 
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(16) float dotProductVector[4];
 
    _mm_store_ps(dotProductVector,
                 dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
 
    number = eighthPoints * 8;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
#endif /*LV_HAVE_SSE && LV_HAVE_MMX*/
 
 
#if LV_HAVE_AVX2 && LV_HAVE_FMA
 
static inline void volk_16i_32fc_dot_prod_32fc_u_avx2_fma(lv_32fc_t* result,
                                                          const short* input,
                                                          const lv_32fc_t* taps,
                                                          unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int sixteenthPoints = num_points / 16;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m128i m0, m1;
    __m256i f0, f1;
    __m256 g0, g1, h0, h1, h2, h3;
    __m256 a0Val, a1Val, a2Val, a3Val;
    __m256 b0Val, b1Val, b2Val, b3Val;
 
    __m256 dotProdVal0 = _mm256_setzero_ps();
    __m256 dotProdVal1 = _mm256_setzero_ps();
    __m256 dotProdVal2 = _mm256_setzero_ps();
    __m256 dotProdVal3 = _mm256_setzero_ps();
 
    for (; number < sixteenthPoints; number++) {
 
        m0 = _mm_loadu_si128((__m128i const*)aPtr);
        m1 = _mm_loadu_si128((__m128i const*)(aPtr + 8));
 
        f0 = _mm256_cvtepi16_epi32(m0);
        g0 = _mm256_cvtepi32_ps(f0);
        f1 = _mm256_cvtepi16_epi32(m1);
        g1 = _mm256_cvtepi32_ps(f1);
 
        h0 = _mm256_unpacklo_ps(g0, g0);
        h1 = _mm256_unpackhi_ps(g0, g0);
        h2 = _mm256_unpacklo_ps(g1, g1);
        h3 = _mm256_unpackhi_ps(g1, g1);
 
        a0Val = _mm256_permute2f128_ps(h0, h1, 0x20);
        a1Val = _mm256_permute2f128_ps(h0, h1, 0x31);
        a2Val = _mm256_permute2f128_ps(h2, h3, 0x20);
        a3Val = _mm256_permute2f128_ps(h2, h3, 0x31);
 
        b0Val = _mm256_loadu_ps(bPtr);
        b1Val = _mm256_loadu_ps(bPtr + 8);
        b2Val = _mm256_loadu_ps(bPtr + 16);
        b3Val = _mm256_loadu_ps(bPtr + 24);
 
        dotProdVal0 = _mm256_fmadd_ps(a0Val, b0Val, dotProdVal0);
        dotProdVal1 = _mm256_fmadd_ps(a1Val, b1Val, dotProdVal1);
        dotProdVal2 = _mm256_fmadd_ps(a2Val, b2Val, dotProdVal2);
        dotProdVal3 = _mm256_fmadd_ps(a3Val, b3Val, dotProdVal3);
 
        aPtr += 16;
        bPtr += 32;
    }
 
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
 
    _mm256_store_ps(dotProductVector,
                    dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
    returnValue += lv_cmake(dotProductVector[4], dotProductVector[5]);
    returnValue += lv_cmake(dotProductVector[6], dotProductVector[7]);
 
    number = sixteenthPoints * 16;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
#endif /*LV_HAVE_AVX2 && lV_HAVE_FMA*/
 
 
#ifdef LV_HAVE_AVX2
 
static inline void volk_16i_32fc_dot_prod_32fc_u_avx2(lv_32fc_t* result,
                                                      const short* input,
                                                      const lv_32fc_t* taps,
                                                      unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int sixteenthPoints = num_points / 16;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m128i m0, m1;
    __m256i f0, f1;
    __m256 g0, g1, h0, h1, h2, h3;
    __m256 a0Val, a1Val, a2Val, a3Val;
    __m256 b0Val, b1Val, b2Val, b3Val;
    __m256 c0Val, c1Val, c2Val, c3Val;
 
    __m256 dotProdVal0 = _mm256_setzero_ps();
    __m256 dotProdVal1 = _mm256_setzero_ps();
    __m256 dotProdVal2 = _mm256_setzero_ps();
    __m256 dotProdVal3 = _mm256_setzero_ps();
 
    for (; number < sixteenthPoints; number++) {
 
        m0 = _mm_loadu_si128((__m128i const*)aPtr);
        m1 = _mm_loadu_si128((__m128i const*)(aPtr + 8));
 
        f0 = _mm256_cvtepi16_epi32(m0);
        g0 = _mm256_cvtepi32_ps(f0);
        f1 = _mm256_cvtepi16_epi32(m1);
        g1 = _mm256_cvtepi32_ps(f1);
 
        h0 = _mm256_unpacklo_ps(g0, g0);
        h1 = _mm256_unpackhi_ps(g0, g0);
        h2 = _mm256_unpacklo_ps(g1, g1);
        h3 = _mm256_unpackhi_ps(g1, g1);
 
        a0Val = _mm256_permute2f128_ps(h0, h1, 0x20);
        a1Val = _mm256_permute2f128_ps(h0, h1, 0x31);
        a2Val = _mm256_permute2f128_ps(h2, h3, 0x20);
        a3Val = _mm256_permute2f128_ps(h2, h3, 0x31);
 
        b0Val = _mm256_loadu_ps(bPtr);
        b1Val = _mm256_loadu_ps(bPtr + 8);
        b2Val = _mm256_loadu_ps(bPtr + 16);
        b3Val = _mm256_loadu_ps(bPtr + 24);
 
        c0Val = _mm256_mul_ps(a0Val, b0Val);
        c1Val = _mm256_mul_ps(a1Val, b1Val);
        c2Val = _mm256_mul_ps(a2Val, b2Val);
        c3Val = _mm256_mul_ps(a3Val, b3Val);
 
        dotProdVal0 = _mm256_add_ps(c0Val, dotProdVal0);
        dotProdVal1 = _mm256_add_ps(c1Val, dotProdVal1);
        dotProdVal2 = _mm256_add_ps(c2Val, dotProdVal2);
        dotProdVal3 = _mm256_add_ps(c3Val, dotProdVal3);
 
        aPtr += 16;
        bPtr += 32;
    }
 
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
 
    _mm256_store_ps(dotProductVector,
                    dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
    returnValue += lv_cmake(dotProductVector[4], dotProductVector[5]);
    returnValue += lv_cmake(dotProductVector[6], dotProductVector[7]);
 
    number = sixteenthPoints * 16;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
#endif /*LV_HAVE_AVX2*/
 
 
#if LV_HAVE_SSE && LV_HAVE_MMX
 
 
static inline void volk_16i_32fc_dot_prod_32fc_a_sse(lv_32fc_t* result,
                                                     const short* input,
                                                     const lv_32fc_t* taps,
                                                     unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int eighthPoints = num_points / 8;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m64 m0, m1;
    __m128 f0, f1, f2, f3;
    __m128 a0Val, a1Val, a2Val, a3Val;
    __m128 b0Val, b1Val, b2Val, b3Val;
    __m128 c0Val, c1Val, c2Val, c3Val;
 
    __m128 dotProdVal0 = _mm_setzero_ps();
    __m128 dotProdVal1 = _mm_setzero_ps();
    __m128 dotProdVal2 = _mm_setzero_ps();
    __m128 dotProdVal3 = _mm_setzero_ps();
 
    for (; number < eighthPoints; number++) {
 
        m0 = _mm_set_pi16(*(aPtr + 3), *(aPtr + 2), *(aPtr + 1), *(aPtr + 0));
        m1 = _mm_set_pi16(*(aPtr + 7), *(aPtr + 6), *(aPtr + 5), *(aPtr + 4));
        f0 = _mm_cvtpi16_ps(m0);
        f1 = _mm_cvtpi16_ps(m0);
        f2 = _mm_cvtpi16_ps(m1);
        f3 = _mm_cvtpi16_ps(m1);
 
        a0Val = _mm_unpacklo_ps(f0, f1);
        a1Val = _mm_unpackhi_ps(f0, f1);
        a2Val = _mm_unpacklo_ps(f2, f3);
        a3Val = _mm_unpackhi_ps(f2, f3);
 
        b0Val = _mm_load_ps(bPtr);
        b1Val = _mm_load_ps(bPtr + 4);
        b2Val = _mm_load_ps(bPtr + 8);
        b3Val = _mm_load_ps(bPtr + 12);
 
        c0Val = _mm_mul_ps(a0Val, b0Val);
        c1Val = _mm_mul_ps(a1Val, b1Val);
        c2Val = _mm_mul_ps(a2Val, b2Val);
        c3Val = _mm_mul_ps(a3Val, b3Val);
 
        dotProdVal0 = _mm_add_ps(c0Val, dotProdVal0);
        dotProdVal1 = _mm_add_ps(c1Val, dotProdVal1);
        dotProdVal2 = _mm_add_ps(c2Val, dotProdVal2);
        dotProdVal3 = _mm_add_ps(c3Val, dotProdVal3);
 
        aPtr += 8;
        bPtr += 16;
    }
 
    _mm_empty(); // clear the mmx technology state
 
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(16) float dotProductVector[4];
 
    _mm_store_ps(dotProductVector,
                 dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
 
    number = eighthPoints * 8;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
#endif /*LV_HAVE_SSE && LV_HAVE_MMX*/
 
#ifdef LV_HAVE_AVX2
 
static inline void volk_16i_32fc_dot_prod_32fc_a_avx2(lv_32fc_t* result,
                                                      const short* input,
                                                      const lv_32fc_t* taps,
                                                      unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int sixteenthPoints = num_points / 16;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m128i m0, m1;
    __m256i f0, f1;
    __m256 g0, g1, h0, h1, h2, h3;
    __m256 a0Val, a1Val, a2Val, a3Val;
    __m256 b0Val, b1Val, b2Val, b3Val;
    __m256 c0Val, c1Val, c2Val, c3Val;
 
    __m256 dotProdVal0 = _mm256_setzero_ps();
    __m256 dotProdVal1 = _mm256_setzero_ps();
    __m256 dotProdVal2 = _mm256_setzero_ps();
    __m256 dotProdVal3 = _mm256_setzero_ps();
 
    for (; number < sixteenthPoints; number++) {
 
        m0 = _mm_load_si128((__m128i const*)aPtr);
        m1 = _mm_load_si128((__m128i const*)(aPtr + 8));
 
        f0 = _mm256_cvtepi16_epi32(m0);
        g0 = _mm256_cvtepi32_ps(f0);
        f1 = _mm256_cvtepi16_epi32(m1);
        g1 = _mm256_cvtepi32_ps(f1);
 
        h0 = _mm256_unpacklo_ps(g0, g0);
        h1 = _mm256_unpackhi_ps(g0, g0);
        h2 = _mm256_unpacklo_ps(g1, g1);
        h3 = _mm256_unpackhi_ps(g1, g1);
 
        a0Val = _mm256_permute2f128_ps(h0, h1, 0x20);
        a1Val = _mm256_permute2f128_ps(h0, h1, 0x31);
        a2Val = _mm256_permute2f128_ps(h2, h3, 0x20);
        a3Val = _mm256_permute2f128_ps(h2, h3, 0x31);
 
        b0Val = _mm256_load_ps(bPtr);
        b1Val = _mm256_load_ps(bPtr + 8);
        b2Val = _mm256_load_ps(bPtr + 16);
        b3Val = _mm256_load_ps(bPtr + 24);
 
        c0Val = _mm256_mul_ps(a0Val, b0Val);
        c1Val = _mm256_mul_ps(a1Val, b1Val);
        c2Val = _mm256_mul_ps(a2Val, b2Val);
        c3Val = _mm256_mul_ps(a3Val, b3Val);
 
        dotProdVal0 = _mm256_add_ps(c0Val, dotProdVal0);
        dotProdVal1 = _mm256_add_ps(c1Val, dotProdVal1);
        dotProdVal2 = _mm256_add_ps(c2Val, dotProdVal2);
        dotProdVal3 = _mm256_add_ps(c3Val, dotProdVal3);
 
        aPtr += 16;
        bPtr += 32;
    }
 
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
 
    _mm256_store_ps(dotProductVector,
                    dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
    returnValue += lv_cmake(dotProductVector[4], dotProductVector[5]);
    returnValue += lv_cmake(dotProductVector[6], dotProductVector[7]);
 
    number = sixteenthPoints * 16;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
 
#endif /*LV_HAVE_AVX2*/
 
#if LV_HAVE_AVX2 && LV_HAVE_FMA
 
static inline void volk_16i_32fc_dot_prod_32fc_a_avx2_fma(lv_32fc_t* result,
                                                          const short* input,
                                                          const lv_32fc_t* taps,
                                                          unsigned int num_points)
{
 
    unsigned int number = 0;
    const unsigned int sixteenthPoints = num_points / 16;
 
    lv_32fc_t returnValue = lv_cmake(0.0f, 0.0f);
    const short* aPtr = input;
    const float* bPtr = (float*)taps;
 
    __m128i m0, m1;
    __m256i f0, f1;
    __m256 g0, g1, h0, h1, h2, h3;
    __m256 a0Val, a1Val, a2Val, a3Val;
    __m256 b0Val, b1Val, b2Val, b3Val;
 
    __m256 dotProdVal0 = _mm256_setzero_ps();
    __m256 dotProdVal1 = _mm256_setzero_ps();
    __m256 dotProdVal2 = _mm256_setzero_ps();
    __m256 dotProdVal3 = _mm256_setzero_ps();
 
    for (; number < sixteenthPoints; number++) {
 
        m0 = _mm_load_si128((__m128i const*)aPtr);
        m1 = _mm_load_si128((__m128i const*)(aPtr + 8));
 
        f0 = _mm256_cvtepi16_epi32(m0);
        g0 = _mm256_cvtepi32_ps(f0);
        f1 = _mm256_cvtepi16_epi32(m1);
        g1 = _mm256_cvtepi32_ps(f1);
 
        h0 = _mm256_unpacklo_ps(g0, g0);
        h1 = _mm256_unpackhi_ps(g0, g0);
        h2 = _mm256_unpacklo_ps(g1, g1);
        h3 = _mm256_unpackhi_ps(g1, g1);
 
        a0Val = _mm256_permute2f128_ps(h0, h1, 0x20);
        a1Val = _mm256_permute2f128_ps(h0, h1, 0x31);
        a2Val = _mm256_permute2f128_ps(h2, h3, 0x20);
        a3Val = _mm256_permute2f128_ps(h2, h3, 0x31);
 
        b0Val = _mm256_load_ps(bPtr);
        b1Val = _mm256_load_ps(bPtr + 8);
        b2Val = _mm256_load_ps(bPtr + 16);
        b3Val = _mm256_load_ps(bPtr + 24);
 
        dotProdVal0 = _mm256_fmadd_ps(a0Val, b0Val, dotProdVal0);
        dotProdVal1 = _mm256_fmadd_ps(a1Val, b1Val, dotProdVal1);
        dotProdVal2 = _mm256_fmadd_ps(a2Val, b2Val, dotProdVal2);
        dotProdVal3 = _mm256_fmadd_ps(a3Val, b3Val, dotProdVal3);
 
        aPtr += 16;
        bPtr += 32;
    }
 
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal1);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal2);
    dotProdVal0 = _mm256_add_ps(dotProdVal0, dotProdVal3);
 
    __VOLK_ATTR_ALIGNED(32) float dotProductVector[8];
 
    _mm256_store_ps(dotProductVector,
                    dotProdVal0); // Store the results back into the dot product vector
 
    returnValue += lv_cmake(dotProductVector[0], dotProductVector[1]);
    returnValue += lv_cmake(dotProductVector[2], dotProductVector[3]);
    returnValue += lv_cmake(dotProductVector[4], dotProductVector[5]);
    returnValue += lv_cmake(dotProductVector[6], dotProductVector[7]);
 
    number = sixteenthPoints * 16;
    for (; number < num_points; number++) {
        returnValue += lv_cmake(aPtr[0] * bPtr[0], aPtr[0] * bPtr[1]);
        aPtr += 1;
        bPtr += 2;
    }
 
    *result = returnValue;
}
 
 
#endif /*LV_HAVE_AVX2 && LV_HAVE_FMA*/
 
#ifdef LV_HAVE_RVV
#include <riscv_vector.h>
#include <volk/volk_rvv_intrinsics.h>
 
static inline void volk_16i_32fc_dot_prod_32fc_rvv(lv_32fc_t* result,
                                                   const short* input,
                                                   const lv_32fc_t* taps,
                                                   unsigned int num_points)
{
    vfloat32m4_t vsumr = __riscv_vfmv_v_f_f32m4(0, __riscv_vsetvlmax_e32m4());
    vfloat32m4_t vsumi = vsumr;
    size_t n = num_points;
    for (size_t vl; n > 0; n -= vl, input += vl, taps += vl) {
        vl = __riscv_vsetvl_e32m4(n);
        vuint64m8_t vc = __riscv_vle64_v_u64m8((const uint64_t*)taps, vl);
        vfloat32m4_t vr = __riscv_vreinterpret_f32m4(__riscv_vnsrl(vc, 0, vl));
        vfloat32m4_t vi = __riscv_vreinterpret_f32m4(__riscv_vnsrl(vc, 32, vl));
        vfloat32m4_t v =
            __riscv_vfwcvt_f(__riscv_vle16_v_i16m2((const int16_t*)input, vl), vl);
        vsumr = __riscv_vfmacc_tu(vsumr, vr, v, vl);
        vsumi = __riscv_vfmacc_tu(vsumi, vi, v, vl);
    }
    size_t vl = __riscv_vsetvlmax_e32m1();
    vfloat32m1_t vr = RISCV_SHRINK4(vfadd, f, 32, vsumr);
    vfloat32m1_t vi = RISCV_SHRINK4(vfadd, f, 32, vsumi);
    vfloat32m1_t z = __riscv_vfmv_s_f_f32m1(0, vl);
    *result = lv_cmake(__riscv_vfmv_f(__riscv_vfredusum(vr, z, vl)),
                       __riscv_vfmv_f(__riscv_vfredusum(vi, z, vl)));
}
#endif /*LV_HAVE_RVV*/
 
#ifdef LV_HAVE_RVVSEG
#include <riscv_vector.h>
#include <volk/volk_rvv_intrinsics.h>
 
static inline void volk_16i_32fc_dot_prod_32fc_rvvseg(lv_32fc_t* result,
                                                      const short* input,
                                                      const lv_32fc_t* taps,
                                                      unsigned int num_points)
{
    vfloat32m4_t vsumr = __riscv_vfmv_v_f_f32m4(0, __riscv_vsetvlmax_e32m4());
    vfloat32m4_t vsumi = vsumr;
    size_t n = num_points;
    for (size_t vl; n > 0; n -= vl, input += vl, taps += vl) {
        vl = __riscv_vsetvl_e32m4(n);
        vfloat32m4x2_t vc = __riscv_vlseg2e32_v_f32m4x2((const float*)taps, vl);
        vfloat32m4_t vr = __riscv_vget_f32m4(vc, 0);
        vfloat32m4_t vi = __riscv_vget_f32m4(vc, 1);
        vfloat32m4_t v =
            __riscv_vfwcvt_f(__riscv_vle16_v_i16m2((const int16_t*)input, vl), vl);
        vsumr = __riscv_vfmacc_tu(vsumr, vr, v, vl);
        vsumi = __riscv_vfmacc_tu(vsumi, vi, v, vl);
    }
    size_t vl = __riscv_vsetvlmax_e32m1();
    vfloat32m1_t vr = RISCV_SHRINK4(vfadd, f, 32, vsumr);
    vfloat32m1_t vi = RISCV_SHRINK4(vfadd, f, 32, vsumi);
    vfloat32m1_t z = __riscv_vfmv_s_f_f32m1(0, vl);
    *result = lv_cmake(__riscv_vfmv_f(__riscv_vfredusum(vr, z, vl)),
                       __riscv_vfmv_f(__riscv_vfredusum(vi, z, vl)));
}
#endif /*LV_HAVE_RVVSEG*/
 
#endif /*INCLUDED_volk_16i_32fc_dot_prod_32fc_H*/