Merge commit 'b57e38f52cc3f31a27105c28887d57cd6812c3eb'

* commit 'b57e38f52cc3f31a27105c28887d57cd6812c3eb':
  ac3dsp: x86: Replace inline asm for in-decoder downmixing with standalone asm

Merged-by: Clément Bœsch <u@pkh.me>

libavcodec/ac3dec.c

@@ -1430,19 +1430,19 @@ static int decode_audio_block(AC3DecodeContext *s, int blk)
             ac3_downmix_c_fixed16(s->outptr, s->downmix_coeffs,
                               s->out_channels, s->fbw_channels, 256);
 #else
-            s->ac3dsp.downmix(s->outptr, s->downmix_coeffs,
+            ff_ac3dsp_downmix(&s->ac3dsp, s->outptr, s->downmix_coeffs,
                               s->out_channels, s->fbw_channels, 256);
 #endif
         }
     } else {
         if (downmix_output) {
-            s->ac3dsp.AC3_RENAME(downmix)(s->xcfptr + 1, s->downmix_coeffs,
+            AC3_RENAME(ff_ac3dsp_downmix)(&s->ac3dsp, s->xcfptr + 1, s->downmix_coeffs,
                                           s->out_channels, s->fbw_channels, 256);
         }
 
         if (downmix_output && !s->downmixed) {
             s->downmixed = 1;
-            s->ac3dsp.AC3_RENAME(downmix)(s->dlyptr, s->downmix_coeffs,
+            AC3_RENAME(ff_ac3dsp_downmix)(&s->ac3dsp, s->dlyptr, s->downmix_coeffs,
                                           s->out_channels, s->fbw_channels, 128);
         }
 

libavcodec/ac3dsp.c

@@ -213,49 +213,53 @@ static void ac3_sum_square_butterfly_float_c(float sum[4],
     }
 }
 
+static void ac3_downmix_5_to_2_symmetric_c(float **samples, float **matrix,
+                                           int len)
+{
+    int i;
+    float v0, v1;
+    float front_mix    = matrix[0][0];
+    float center_mix   = matrix[0][1];
+    float surround_mix = matrix[0][3];
+
+    for (i = 0; i < len; i++) {
+        v0 = samples[0][i] * front_mix  +
+             samples[1][i] * center_mix +
+             samples[3][i] * surround_mix;
+
+        v1 = samples[1][i] * center_mix +
+             samples[2][i] * front_mix  +
+             samples[4][i] * surround_mix;
+
+        samples[0][i] = v0;
+        samples[1][i] = v1;
+    }
+}
+
+static void ac3_downmix_5_to_1_symmetric_c(float **samples, float **matrix,
+                                           int len)
+{
+    int i;
+    float front_mix    = matrix[0][0];
+    float center_mix   = matrix[0][1];
+    float surround_mix = matrix[0][3];
+
+    for (i = 0; i < len; i++) {
+        samples[0][i] = samples[0][i] * front_mix    +
+                        samples[1][i] * center_mix   +
+                        samples[2][i] * front_mix    +
+                        samples[3][i] * surround_mix +
+                        samples[4][i] * surround_mix;
+    }
+}
+
 static void ac3_downmix_c(float **samples, float **matrix,
                           int out_ch, int in_ch, int len)
 {
-    int **matrix_cmp = (int **)matrix;
     int i, j;
     float v0, v1;
 
-    if (in_ch == 5 && out_ch == 2 &&
+    if (out_ch == 2) {
-        !(matrix_cmp[1][0] | matrix_cmp[0][2]   |
-          matrix_cmp[1][3] | matrix_cmp[0][4]   |
-         (matrix_cmp[0][1] ^ matrix_cmp[1][1]) |
-         (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) {
-        float front_mix    = matrix[0][0];
-        float center_mix   = matrix[0][1];
-        float surround_mix = matrix[0][3];
-
-        for (i = 0; i < len; i++) {
-            v0 = samples[0][i] * front_mix  +
-                 samples[1][i] * center_mix +
-                 samples[3][i] * surround_mix;
-
-            v1 = samples[1][i] * center_mix +
-                 samples[2][i] * front_mix  +
-                 samples[4][i] * surround_mix;
-
-            samples[0][i] = v0;
-            samples[1][i] = v1;
-        }
-    } else if (in_ch == 5 && out_ch == 1 &&
-               matrix_cmp[0][0] == matrix_cmp[0][2] &&
-               matrix_cmp[0][3] == matrix_cmp[0][4]) {
-        float front_mix    = matrix[0][0];
-        float center_mix   = matrix[0][1];
-        float surround_mix = matrix[0][3];
-
-        for (i = 0; i < len; i++) {
-            samples[0][i] = samples[0][i] * front_mix    +
-                            samples[1][i] * center_mix   +
-                            samples[2][i] * front_mix    +
-                            samples[3][i] * surround_mix +
-                            samples[4][i] * surround_mix;
-        }
-    } else if (out_ch == 2) {
         for (i = 0; i < len; i++) {
             v0 = v1 = 0.0f;
             for (j = 0; j < in_ch; j++) {
@@ -300,6 +304,15 @@ static void ac3_downmix_c_fixed(int32_t **samples, int16_t **matrix,
     }
 }
 
+void ff_ac3dsp_downmix_fixed(AC3DSPContext *c, int32_t **samples, int16_t **matrix,
+                             int out_ch, int in_ch, int len)
+{
+    if (c->downmix_fixed)
+        c->downmix_fixed(samples, matrix, len);
+    else
+        ac3_downmix_c_fixed(samples, matrix, out_ch, in_ch, len);
+}
+
 static void apply_window_int16_c(int16_t *output, const int16_t *input,
                                  const int16_t *window, unsigned int len)
 {
@@ -313,6 +326,38 @@ static void apply_window_int16_c(int16_t *output, const int16_t *input,
     }
 }
 
+void ff_ac3dsp_downmix(AC3DSPContext *c, float **samples, float **matrix,
+                       int out_ch, int in_ch, int len)
+{
+    if (c->in_channels != in_ch || c->out_channels != out_ch) {
+        int **matrix_cmp = (int **)matrix;
+
+        c->in_channels  = in_ch;
+        c->out_channels = out_ch;
+        c->downmix      = NULL;
+
+        if (in_ch == 5 && out_ch == 2 &&
+            !(matrix_cmp[1][0] | matrix_cmp[0][2]   |
+              matrix_cmp[1][3] | matrix_cmp[0][4]   |
+             (matrix_cmp[0][1] ^ matrix_cmp[1][1]) |
+             (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) {
+            c->downmix = ac3_downmix_5_to_2_symmetric_c;
+        } else if (in_ch == 5 && out_ch == 1 &&
+                   matrix_cmp[0][0] == matrix_cmp[0][2] &&
+                   matrix_cmp[0][3] == matrix_cmp[0][4]) {
+            c->downmix = ac3_downmix_5_to_1_symmetric_c;
+        }
+
+        if (ARCH_X86)
+            ff_ac3dsp_set_downmix_x86(c);
+    }
+
+    if (c->downmix)
+        c->downmix(samples, matrix, len);
+    else
+        ac3_downmix_c(samples, matrix, out_ch, in_ch, len);
+}
+
 av_cold void ff_ac3dsp_init(AC3DSPContext *c, int bit_exact)
 {
     c->ac3_exponent_min = ac3_exponent_min_c;
@@ -326,8 +371,10 @@ av_cold void ff_ac3dsp_init(AC3DSPContext *c, int bit_exact)
     c->extract_exponents = ac3_extract_exponents_c;
     c->sum_square_butterfly_int32 = ac3_sum_square_butterfly_int32_c;
     c->sum_square_butterfly_float = ac3_sum_square_butterfly_float_c;
-    c->downmix = ac3_downmix_c;
+    c->in_channels           = 0;
-    c->downmix_fixed = ac3_downmix_c_fixed;
+    c->out_channels          = 0;
+    c->downmix               = NULL;
+    c->downmix_fixed         = NULL;
     c->apply_window_int16 = apply_window_int16_c;
 
     if (ARCH_ARM)

libavcodec/ac3dsp.h

@@ -132,11 +132,10 @@ typedef struct AC3DSPContext {
     void (*sum_square_butterfly_float)(float sum[4], const float *coef0,
                                        const float *coef1, int len);
 
-    void (*downmix)(float **samples, float **matrix, int out_ch,
+    int out_channels;
-                    int in_ch, int len);
+    int in_channels;
-
+    void (*downmix)(float **samples, float **matrix, int len);
-    void (*downmix_fixed)(int32_t **samples, int16_t **matrix, int out_ch,
+    void (*downmix_fixed)(int32_t **samples, int16_t **matrix, int len);
-                          int in_ch, int len);
 
     /**
      * Apply symmetric window in 16-bit fixed-point.
@@ -158,4 +157,11 @@ void ff_ac3dsp_init_arm(AC3DSPContext *c, int bit_exact);
 void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact);
 void ff_ac3dsp_init_mips(AC3DSPContext *c, int bit_exact);
 
+void ff_ac3dsp_downmix(AC3DSPContext *c, float **samples, float **matrix,
+                       int out_ch, int in_ch, int len);
+void ff_ac3dsp_downmix_fixed(AC3DSPContext *c, int32_t **samples, int16_t **matrix,
+                             int out_ch, int in_ch, int len);
+
+void ff_ac3dsp_set_downmix_x86(AC3DSPContext *c);
+
 #endif /* AVCODEC_AC3DSP_H */

libavcodec/x86/Makefile

@@ -87,7 +87,8 @@ MMX-OBJS-$(CONFIG_SNOW_DECODER)        += x86/snowdsp.o
 MMX-OBJS-$(CONFIG_SNOW_ENCODER)        += x86/snowdsp.o
 
 # subsystems
-YASM-OBJS-$(CONFIG_AC3DSP)             += x86/ac3dsp.o
+YASM-OBJS-$(CONFIG_AC3DSP)             += x86/ac3dsp.o                  \
+                                          x86/ac3dsp_downmix.o
 YASM-OBJS-$(CONFIG_AUDIODSP)           += x86/audiodsp.o
 YASM-OBJS-$(CONFIG_BLOCKDSP)           += x86/blockdsp.o
 YASM-OBJS-$(CONFIG_BSWAPDSP)           += x86/bswapdsp.o

libavcodec/x86/ac3dsp_downmix.asm

@@ -0,0 +1,187 @@
+;*****************************************************************************
+;* x86-optimized AC-3 downmixing
+;* Copyright (c) 2012 Justin Ruggles
+;*
+;* This file is part of FFmpeg.
+;*
+;* FFmpeg is free software; you can redistribute it and/or
+;* modify it under the terms of the GNU Lesser General Public
+;* License as published by the Free Software Foundation; either
+;* version 2.1 of the License, or (at your option) any later version.
+;*
+;* FFmpeg is distributed in the hope that it will be useful,
+;* but WITHOUT ANY WARRANTY; without even the implied warranty of
+;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;* Lesser General Public License for more details.
+;*
+;* You should have received a copy of the GNU Lesser General Public
+;* License along with FFmpeg; if not, write to the Free Software
+;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+;******************************************************************************
+
+;******************************************************************************
+;* This is based on the channel mixing asm in libavresample, but it is
+;* simplified for only float coefficients and only 3 to 6 channels.
+;******************************************************************************
+
+%include "libavutil/x86/x86util.asm"
+
+SECTION .text
+
+;-----------------------------------------------------------------------------
+; functions to downmix from 3 to 6 channels to mono or stereo
+; void ff_ac3_downmix_*(float **samples, float **matrix, int len);
+;-----------------------------------------------------------------------------
+
+%macro AC3_DOWNMIX 2 ; %1 = in channels, %2 = out channels
+; define some names to make the code clearer
+%assign  in_channels %1
+%assign out_channels %2
+%assign stereo out_channels - 1
+
+; determine how many matrix elements must go on the stack vs. mmregs
+%assign matrix_elements in_channels * out_channels
+%if stereo
+    %assign needed_mmregs 4
+%else
+    %assign needed_mmregs 3
+%endif
+%assign matrix_elements_mm num_mmregs - needed_mmregs
+%if matrix_elements < matrix_elements_mm
+    %assign matrix_elements_mm matrix_elements
+%endif
+%assign total_mmregs needed_mmregs+matrix_elements_mm
+%if matrix_elements_mm < matrix_elements
+    %assign matrix_elements_stack matrix_elements - matrix_elements_mm
+%else
+    %assign matrix_elements_stack 0
+%endif
+
+cglobal ac3_downmix_%1_to_%2, 3,in_channels+1,total_mmregs,0-matrix_elements_stack*mmsize, src0, src1, len, src2, src3, src4, src5
+
+; load matrix pointers
+%define matrix0q r1q
+%define matrix1q r3q
+%if stereo
+    mov      matrix1q, [matrix0q+gprsize]
+%endif
+    mov      matrix0q, [matrix0q]
+
+; define matrix coeff names
+%assign %%i 0
+%assign %%j needed_mmregs
+%rep in_channels
+    %if %%i >= matrix_elements_mm
+        CAT_XDEFINE mx_stack_0_, %%i, 1
+        CAT_XDEFINE mx_0_, %%i, [rsp+(%%i-matrix_elements_mm)*mmsize]
+    %else
+        CAT_XDEFINE mx_stack_0_, %%i, 0
+        CAT_XDEFINE mx_0_, %%i, m %+ %%j
+        %assign %%j %%j+1
+    %endif
+    %assign %%i %%i+1
+%endrep
+%if stereo
+%assign %%i 0
+%rep in_channels
+    %if in_channels + %%i >= matrix_elements_mm
+        CAT_XDEFINE mx_stack_1_, %%i, 1
+        CAT_XDEFINE mx_1_, %%i, [rsp+(in_channels+%%i-matrix_elements_mm)*mmsize]
+    %else
+        CAT_XDEFINE mx_stack_1_, %%i, 0
+        CAT_XDEFINE mx_1_, %%i, m %+ %%j
+        %assign %%j %%j+1
+    %endif
+    %assign %%i %%i+1
+%endrep
+%endif
+
+; load/splat matrix coeffs
+%assign %%i 0
+%rep in_channels
+    %if mx_stack_0_ %+ %%i
+        VBROADCASTSS m0, [matrix0q+4*%%i]
+        mova  mx_0_ %+ %%i, m0
+    %else
+        VBROADCASTSS mx_0_ %+ %%i, [matrix0q+4*%%i]
+    %endif
+    %if stereo
+    %if mx_stack_1_ %+ %%i
+        VBROADCASTSS m0, [matrix1q+4*%%i]
+        mova  mx_1_ %+ %%i, m0
+    %else
+        VBROADCASTSS mx_1_ %+ %%i, [matrix1q+4*%%i]
+    %endif
+    %endif
+    %assign %%i %%i+1
+%endrep
+
+    lea          lenq, [4*r2d]
+    ; load channel pointers to registers
+%assign %%i 1
+%rep (in_channels - 1)
+    mov         src %+ %%i %+ q, [src0q+%%i*gprsize]
+    add         src %+ %%i %+ q, lenq
+    %assign %%i %%i+1
+%endrep
+    mov         src0q, [src0q]
+    add         src0q, lenq
+    neg          lenq
+.loop:
+    %if stereo || mx_stack_0_0
+    mova           m0, [src0q+lenq]
+    %endif
+    %if stereo
+    mulps          m1, m0, mx_1_0
+    %endif
+    %if stereo || mx_stack_0_0
+    mulps          m0, m0, mx_0_0
+    %else
+    mulps          m0, mx_0_0, [src0q+lenq]
+    %endif
+%assign %%i 1
+%rep (in_channels - 1)
+    %define src_ptr src %+ %%i %+ q
+    ; avoid extra load for mono if matrix is in a mm register
+    %if stereo || mx_stack_0_ %+ %%i
+    mova           m2, [src_ptr+lenq]
+    %endif
+    %if stereo
+    FMULADD_PS     m1, m2, mx_1_ %+ %%i, m1, m3
+    %endif
+    %if stereo || mx_stack_0_ %+ %%i
+    FMULADD_PS     m0, m2, mx_0_ %+ %%i, m0, m2
+    %else
+    FMULADD_PS     m0, mx_0_ %+ %%i, [src_ptr+lenq], m0, m1
+    %endif
+    %assign %%i %%i+1
+%endrep
+    mova [src0q+lenq], m0
+    %if stereo
+    mova [src1q+lenq], m1
+    %endif
+
+    add          lenq, mmsize
+    jl .loop
+    RET
+%endmacro
+
+%macro AC3_DOWNMIX_FUNCS 0
+%assign %%i 3
+%rep 4
+    INIT_XMM sse
+    AC3_DOWNMIX %%i, 1
+    AC3_DOWNMIX %%i, 2
+    INIT_YMM avx
+    AC3_DOWNMIX %%i, 1
+    AC3_DOWNMIX %%i, 2
+    %if HAVE_FMA3_EXTERNAL
+    INIT_YMM fma3
+    AC3_DOWNMIX %%i, 1
+    AC3_DOWNMIX %%i, 2
+    %endif
+    %assign %%i %%i+1
+%endrep
+%endmacro
+
+AC3_DOWNMIX_FUNCS

libavcodec/x86/ac3dsp_init.c

@@ -63,140 +63,6 @@ void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
                                       const int16_t *window, unsigned int len);
 
-#if ARCH_X86_32 && defined(__INTEL_COMPILER)
-#       undef HAVE_7REGS
-#       define HAVE_7REGS 0
-#endif
-
-#if HAVE_SSE_INLINE && HAVE_7REGS
-
-#define IF1(x) x
-#define IF0(x)
-
-#define MIX5(mono, stereo)                                      \
-    __asm__ volatile (                                          \
-        "movss           0(%1), %%xmm5          \n"             \
-        "movss           4(%1), %%xmm6          \n"             \
-        "movss          12(%1), %%xmm7          \n"             \
-        "shufps     $0, %%xmm5, %%xmm5          \n"             \
-        "shufps     $0, %%xmm6, %%xmm6          \n"             \
-        "shufps     $0, %%xmm7, %%xmm7          \n"             \
-        "1:                                     \n"             \
-        "movaps       (%0, %2), %%xmm0          \n"             \
-        "movaps       (%0, %3), %%xmm1          \n"             \
-        "movaps       (%0, %4), %%xmm2          \n"             \
-        "movaps       (%0, %5), %%xmm3          \n"             \
-        "movaps       (%0, %6), %%xmm4          \n"             \
-        "mulps          %%xmm5, %%xmm0          \n"             \
-        "mulps          %%xmm6, %%xmm1          \n"             \
-        "mulps          %%xmm5, %%xmm2          \n"             \
-        "mulps          %%xmm7, %%xmm3          \n"             \
-        "mulps          %%xmm7, %%xmm4          \n"             \
- stereo("addps          %%xmm1, %%xmm0          \n")            \
-        "addps          %%xmm1, %%xmm2          \n"             \
-        "addps          %%xmm3, %%xmm0          \n"             \
-        "addps          %%xmm4, %%xmm2          \n"             \
-   mono("addps          %%xmm2, %%xmm0          \n")            \
-        "movaps         %%xmm0, (%0, %2)        \n"             \
- stereo("movaps         %%xmm2, (%0, %3)        \n")            \
-        "add               $16, %0              \n"             \
-        "jl                 1b                  \n"             \
-        : "+&r"(i)                                              \
-        : "r"(matrix[0]),                                          \
-          "r"(samples[0] + len),                                \
-          "r"(samples[1] + len),                                \
-          "r"(samples[2] + len),                                \
-          "r"(samples[3] + len),                                \
-          "r"(samples[4] + len)                                 \
-        : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3",      \
-                      "%xmm4", "%xmm5", "%xmm6", "%xmm7",)      \
-         "memory"                                               \
-    );
-
-#define MIX_MISC(stereo)                                        \
-    __asm__ volatile (                                          \
-        "mov              %5, %2            \n"                 \
-        "1:                                 \n"                 \
-        "mov -%c7(%6, %2, %c8), %3          \n"                 \
-        "movaps     (%3, %0), %%xmm0        \n"                 \
- stereo("movaps       %%xmm0, %%xmm1        \n")                \
-        "mulps        %%xmm4, %%xmm0        \n"                 \
- stereo("mulps        %%xmm5, %%xmm1        \n")                \
-        "2:                                 \n"                 \
-        "mov   (%6, %2, %c8), %1            \n"                 \
-        "movaps     (%1, %0), %%xmm2        \n"                 \
- stereo("movaps       %%xmm2, %%xmm3        \n")                \
-        "mulps   (%4, %2, 8), %%xmm2        \n"                 \
- stereo("mulps 16(%4, %2, 8), %%xmm3        \n")                \
-        "addps        %%xmm2, %%xmm0        \n"                 \
- stereo("addps        %%xmm3, %%xmm1        \n")                \
-        "add              $4, %2            \n"                 \
-        "jl               2b                \n"                 \
-        "mov              %5, %2            \n"                 \
- stereo("mov   (%6, %2, %c8), %1            \n")                \
-        "movaps       %%xmm0, (%3, %0)      \n"                 \
- stereo("movaps       %%xmm1, (%1, %0)      \n")                \
-        "add             $16, %0            \n"                 \
-        "jl               1b                \n"                 \
-        : "+&r"(i), "=&r"(j), "=&r"(k), "=&r"(m)                \
-        : "r"(matrix_simd + in_ch),                             \
-          "g"((intptr_t) - 4 * (in_ch - 1)),                    \
-          "r"(samp + in_ch),                                    \
-          "i"(sizeof(float *)), "i"(sizeof(float *)/4)          \
-        : "memory"                                              \
-    );
-
-static void ac3_downmix_sse(float **samples, float **matrix,
-                            int out_ch, int in_ch, int len)
-{
-    int **matrix_cmp = (int **)matrix;
-    intptr_t i, j, k, m;
-
-    i = -len * sizeof(float);
-    if (in_ch == 5 && out_ch == 2 &&
-        !(matrix_cmp[1][0] | matrix_cmp[0][2]   |
-          matrix_cmp[1][3] | matrix_cmp[0][4]   |
-          (matrix_cmp[0][1] ^ matrix_cmp[1][1]) |
-          (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) {
-        MIX5(IF0, IF1);
-    } else if (in_ch == 5 && out_ch == 1 &&
-               matrix_cmp[0][0] == matrix_cmp[0][2] &&
-               matrix_cmp[0][3] == matrix_cmp[0][4]) {
-        MIX5(IF1, IF0);
-    } else {
-        LOCAL_ALIGNED(16, float, matrix_simd, [AC3_MAX_CHANNELS], [2][4]);
-        float *samp[AC3_MAX_CHANNELS];
-
-        for (j = 0; j < in_ch; j++)
-            samp[j] = samples[j] + len;
-
-        j = 2 * in_ch * sizeof(float);
-        k =     in_ch * sizeof(float);
-        __asm__ volatile (
-            "1:                                 \n"
-            "sub             $4, %1             \n"
-            "sub             $8, %0             \n"
-            "movss     (%3, %1), %%xmm4         \n"
-            "movss     (%4, %1), %%xmm5         \n"
-            "shufps          $0, %%xmm4, %%xmm4 \n"
-            "shufps          $0, %%xmm5, %%xmm5 \n"
-            "movaps      %%xmm4,   (%2, %0, 4)  \n"
-            "movaps      %%xmm5, 16(%2, %0, 4)  \n"
-            "jg              1b                 \n"
-            : "+&r"(j), "+&r"(k)
-            : "r"(matrix_simd), "r"(matrix[0]), "r"(matrix[1])
-            : "memory"
-        );
-        if (out_ch == 2) {
-            MIX_MISC(IF1);
-        } else {
-            MIX_MISC(IF0);
-        }
-    }
-}
-
-#endif /* HAVE_SSE_INLINE && HAVE_7REGS */
-
 av_cold void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact)
 {
     int cpu_flags = av_get_cpu_flags();
@@ -252,10 +118,47 @@ av_cold void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact)
             c->apply_window_int16 = ff_apply_window_int16_ssse3;
         }
     }
-
+}
-#if HAVE_SSE_INLINE && HAVE_7REGS
+
-    if (INLINE_SSE(cpu_flags)) {
+#define DOWNMIX_FUNC_OPT(ch, opt)                                       \
-        c->downmix = ac3_downmix_sse;
+void ff_ac3_downmix_ ## ch ## _to_1_ ## opt(float **samples,            \
-    }
+                                            float **matrix, int len);   \
-#endif
+void ff_ac3_downmix_ ## ch ## _to_2_ ## opt(float **samples,            \
+                                            float **matrix, int len);
+
+#define DOWNMIX_FUNCS(opt)   \
+    DOWNMIX_FUNC_OPT(3, opt) \
+    DOWNMIX_FUNC_OPT(4, opt) \
+    DOWNMIX_FUNC_OPT(5, opt) \
+    DOWNMIX_FUNC_OPT(6, opt)
+
+DOWNMIX_FUNCS(sse)
+DOWNMIX_FUNCS(avx)
+DOWNMIX_FUNCS(fma3)
+
+void ff_ac3dsp_set_downmix_x86(AC3DSPContext *c)
+{
+    int cpu_flags = av_get_cpu_flags();
+
+#define SET_DOWNMIX(ch, suf, SUF)                                       \
+    if (ch == c->in_channels) {                                         \
+        if (EXTERNAL_ ## SUF (cpu_flags)) {                             \
+            if (c->out_channels == 1)                                   \
+                c->downmix = ff_ac3_downmix_ ## ch ## _to_1_ ## suf;    \
+            else                                                        \
+                c->downmix = ff_ac3_downmix_ ## ch ## _to_2_ ## suf;    \
+        }                                                               \
+    }
+
+#define SET_DOWNMIX_ALL(suf, SUF)                   \
+    SET_DOWNMIX(3, suf, SUF)                        \
+    SET_DOWNMIX(4, suf, SUF)                        \
+    SET_DOWNMIX(5, suf, SUF)                        \
+    SET_DOWNMIX(6, suf, SUF)
+
+    SET_DOWNMIX_ALL(sse,  SSE)
+    if (!(cpu_flags & AV_CPU_FLAG_AVXSLOW)) {
+        SET_DOWNMIX_ALL(avx,  AVX)
+        SET_DOWNMIX_ALL(fma3, FMA3)
+    }
 }