audio: change playback speed directly in resampler

Although the libraries we use for resampling (libavresample and
libswresample) do not support changing sampelrate on the fly, this makes
it easier to make sure no audio buffers are implicitly dropped. In fact,
this commit adds additional code to drain the resampler explicitly.

Changing speed twice without feeding audio in-between made it crash
with libavresample inc ertain cases (libswresample is fine). This is
probably a libavresample bug. Hopefully this will be fixed, and also I
attempted to workaround the situation that crashes it. (It seems to
point in direction of random memory corruption, though.)

audio/filter/af.c

@@ -54,7 +54,6 @@ extern const struct af_info af_info_center;
 extern const struct af_info af_info_sinesuppress;
 extern const struct af_info af_info_karaoke;
 extern const struct af_info af_info_scaletempo;
-extern const struct af_info af_info_forcespeed;
 extern const struct af_info af_info_bs2b;
 extern const struct af_info af_info_lavfi;
 extern const struct af_info af_info_convert24;
@@ -87,7 +86,6 @@ static const struct af_info *const filter_list[] = {
     &af_info_center,
     &af_info_sinesuppress,
     &af_info_karaoke,
-    &af_info_forcespeed,
     &af_info_scaletempo,
 #if HAVE_LIBBS2B
     &af_info_bs2b,

audio/filter/af_forcespeed.c

@@ -1,71 +0,0 @@
-/*
- * This file is part of mpv.
- *
- * mpv is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * mpv is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with mpv.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "af.h"
-
-struct priv {
-    double speed;
-};
-
-static int control(struct af_instance *af, int cmd, void *arg)
-{
-    struct priv *priv = af->priv;
-
-    switch (cmd) {
-    case AF_CONTROL_REINIT: {
-        struct mp_audio *in = arg;
-        struct mp_audio orig_in = *in;
-        struct mp_audio *out = af->data;
-
-        mp_audio_copy_config(out, in);
-        out->rate = in->rate * priv->speed;
-
-        return mp_audio_config_equals(in, &orig_in) ? AF_OK : AF_FALSE;
-    }
-    case AF_CONTROL_SET_PLAYBACK_SPEED_RESAMPLE: {
-        priv->speed = *(double *)arg;
-        return AF_OK;
-    }
-    }
-    return AF_UNKNOWN;
-}
-
-static int filter(struct af_instance *af, struct mp_audio *data)
-{
-    if (data)
-        mp_audio_copy_config(data, af->data);
-    af_add_output_frame(af, data);
-    return 0;
-}
-
-static int af_open(struct af_instance *af)
-{
-    struct priv *priv = af->priv;
-    af->control = control;
-    af->filter_frame = filter;
-    priv->speed = 1.0;
-    return AF_OK;
-}
-
-#define OPT_BASE_STRUCT struct priv
-
-const struct af_info af_info_forcespeed = {
-    .info = "Force audio speed",
-    .name = "forcespeed",
-    .open = af_open,
-    .priv_size = sizeof(struct priv),
-};

audio/filter/af_lavrresample.c

@@ -24,6 +24,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <inttypes.h>
+#include <math.h>
 #include <assert.h>
 
 #include <libavutil/opt.h>
@@ -32,7 +33,7 @@
 #include <libavutil/samplefmt.h>
 #include <libavutil/mathematics.h>
 
-#include "talloc.h"
+#include "common/common.h"
 #include "config.h"
 
 #if HAVE_LIBAVRESAMPLE
@@ -64,7 +65,8 @@ struct af_resample_opts {
     int linear;
     double cutoff;
 
-    int in_rate;
+    int in_rate_af; // filter input sample rate
+    int in_rate;    // actual rate (used by lavr), adjusted for playback speed
     int in_format;
     struct mp_chmap in_channels;
     int out_rate;
@@ -75,6 +77,9 @@ struct af_resample_opts {
 struct af_resample {
     int allow_detach;
     char **avopts;
+    double playback_speed;
+    struct mp_audio *pending;
+    bool avrctx_ok;
     struct AVAudioResampleContext *avrctx;
     struct AVAudioResampleContext *avrctx_out; // for output channel reordering
     struct af_resample_opts ctx;   // opts in the context
@@ -94,6 +99,10 @@ static void drop_all_output(struct af_resample *s)
 {
     while (avresample_read(s->avrctx, NULL, 1000) > 0) {}
 }
+static int get_drain_samples(struct af_resample *s)
+{
+    return avresample_get_out_samples(s->avrctx, 0);
+}
 #else
 static int get_delay(struct af_resample *s)
 {
@@ -103,18 +112,39 @@ static void drop_all_output(struct af_resample *s)
 {
     while (swr_drop_output(s->avrctx, 1000) > 0) {}
 }
+static int get_drain_samples(struct af_resample *s)
+{
+    return 4096; // libswscale does not have this
+}
 #endif
 
+static int resample_frame(struct AVAudioResampleContext *r,
+                          struct mp_audio *out, struct mp_audio *in)
+{
+    return avresample_convert(r,
+        out ? (uint8_t **)out->planes : NULL,
+        out ? mp_audio_get_allocated_size(out) : 0,
+        out ? out->samples : 0,
+        in ? (uint8_t **)in->planes : NULL,
+        in ? mp_audio_get_allocated_size(in) : 0,
+        in ? in->samples : 0);
+}
+
 static double af_resample_default_cutoff(int filter_size)
 {
     return FFMAX(1.0 - 6.5 / (filter_size + 8), 0.80);
 }
 
+static int rate_from_speed(int rate, double speed)
+{
+    return lrint(rate * speed);
+}
+
 static bool needs_lavrctx_reconfigure(struct af_resample *s,
                                       struct mp_audio *in,
                                       struct mp_audio *out)
 {
-    return s->ctx.in_rate     != in->rate ||
+    return s->ctx.in_rate_af  != in->rate ||
            s->ctx.in_format   != in->format ||
            !mp_chmap_equals(&s->ctx.in_channels, &in->channels) ||
            s->ctx.out_rate    != out->rate ||
@@ -138,6 +168,8 @@ static int configure_lavrr(struct af_instance *af, struct mp_audio *in,
 {
     struct af_resample *s = af->priv;
 
+    s->avrctx_ok = false;
+
     enum AVSampleFormat in_samplefmt = af_to_avformat(in->format);
     enum AVSampleFormat out_samplefmt = af_to_avformat(out->format);
 
@@ -147,8 +179,12 @@ static int configure_lavrr(struct af_instance *af, struct mp_audio *in,
     avresample_close(s->avrctx);
     avresample_close(s->avrctx_out);
 
+    talloc_free(s->pending);
+    s->pending = NULL;
+
     s->ctx.out_rate    = out->rate;
-    s->ctx.in_rate     = in->rate;
+    s->ctx.in_rate_af  = in->rate;
+    s->ctx.in_rate     = rate_from_speed(in->rate, s->playback_speed);
     s->ctx.out_format  = out->format;
     s->ctx.in_format   = in->format;
     s->ctx.out_channels= out->channels;
@@ -217,6 +253,7 @@ static int configure_lavrr(struct af_instance *af, struct mp_audio *in,
         MP_ERR(af, "Cannot open Libavresample Context. \n");
         return AF_ERROR;
     }
+    s->avrctx_ok = true;
     return AF_OK;
 }
 
@@ -234,7 +271,7 @@ static int control(struct af_instance *af, int cmd, void *arg)
         if (((out->rate    == in->rate) || (out->rate == 0)) &&
             (out->format   == in->format) &&
             (mp_chmap_equals(&out->channels, &in->channels) || out->nch == 0) &&
-            s->allow_detach)
+            s->allow_detach && s->playback_speed == 1.0)
             return AF_DETACH;
 
         if (out->rate == 0)
@@ -270,6 +307,26 @@ static int control(struct af_instance *af, int cmd, void *arg)
     case AF_CONTROL_SET_RESAMPLE_RATE:
         out->rate = *(int *)arg;
         return AF_OK;
+    case AF_CONTROL_SET_PLAYBACK_SPEED_RESAMPLE: {
+        s->playback_speed = *(double *)arg;
+        int new_rate = rate_from_speed(s->ctx.in_rate_af, s->playback_speed);
+        if (new_rate != s->ctx.in_rate && s->avrctx_ok && af->fmt_out.format) {
+            // Before reconfiguring, drain the audio that is still buffered
+            // in the resampler.
+            talloc_free(s->pending);
+            s->pending = talloc_zero(NULL, struct mp_audio);
+            mp_audio_copy_config(s->pending, &af->fmt_out);
+            s->pending->samples = get_drain_samples(s);
+            if (s->pending->samples > 0) {
+                mp_audio_realloc_min(s->pending, s->pending->samples);
+                int r = resample_frame(s->avrctx, s->pending, NULL);
+                s->pending->samples = MPMAX(r, 0);
+            }
+            // Reinitialize resampler.
+            configure_lavrr(af, &af->fmt_in, &af->fmt_out);
+        }
+        return AF_OK;
+    }
     case AF_CONTROL_RESET:
         drop_all_output(s);
         return AF_OK;
@@ -289,6 +346,7 @@ static void uninit(struct af_instance *af)
     if (s->avrctx_out)
         avresample_close(s->avrctx_out);
     avresample_free(&s->avrctx_out);
+    talloc_free(s->pending);
 }
 
 static bool needs_reorder(int *reorder, int num_ch)
@@ -309,22 +367,19 @@ static void reorder_planes(struct mp_audio *mpa, int *reorder)
     }
 }
 
-static int resample_frame(struct AVAudioResampleContext *r,
-                          struct mp_audio *out, struct mp_audio *in)
-{
-    return avresample_convert(r,
-        out ? (uint8_t **)out->planes : NULL,
-        out ? mp_audio_get_allocated_size(out) : 0,
-        out ? out->samples : 0,
-        in ? (uint8_t **)in->planes : NULL,
-        in ? mp_audio_get_allocated_size(in) : 0,
-        in ? in->samples : 0);
-}
-
 static int filter(struct af_instance *af, struct mp_audio *in)
 {
     struct af_resample *s = af->priv;
 
+    if (s->pending) {
+        if (s->pending->samples) {
+            af_add_output_frame(af, s->pending);
+        } else {
+            talloc_free(s->pending);
+        }
+        s->pending = NULL;
+    }
+
     int samples = avresample_available(s->avrctx) +
         av_rescale_rnd(get_delay(s) + (in ? in->samples : 0),
                        s->ctx.out_rate, s->ctx.in_rate, AV_ROUND_UP);
@@ -412,6 +467,7 @@ const struct af_info af_info_lavrresample = {
             .cutoff      = 0.0,
             .phase_shift = 10,
         },
+        .playback_speed = 1.0,
         .allow_detach = 1,
     },
     .options = (const struct m_option[]) {

old-makefile

@@ -134,7 +134,6 @@ SOURCES = audio/audio.c \
           audio/filter/af_equalizer.c \
           audio/filter/af_export.c \
           audio/filter/af_extrastereo.c \
-          audio/filter/af_forcespeed.c \
           audio/filter/af_format.c \
           audio/filter/af_hrtf.c \
           audio/filter/af_karaoke.c \

player/audio.c

@@ -66,7 +66,6 @@ static int recreate_audio_filters(struct MPContext *mpctx)
 
     struct MPOpts *opts = mpctx->opts;
     struct af_stream *afs = mpctx->d_audio->afilter;
-    bool need_reinit = false;
 
     double speed = opts->playback_speed;
 
@@ -84,7 +83,7 @@ static int recreate_audio_filters(struct MPContext *mpctx)
             if (!af_control_any_rev(afs, AF_CONTROL_SET_PLAYBACK_SPEED, &speed))
             {
                 char *filter = method == AF_CONTROL_SET_PLAYBACK_SPEED
-                             ? "scaletempo" : "forcespeed";
+                             ? "scaletempo" : "lavrresample";
                 if (try_filter(mpctx, filter, "playback-speed", NULL) < 0)
                     return -1;
                 // Try again.
@@ -94,11 +93,6 @@ static int recreate_audio_filters(struct MPContext *mpctx)
                 method = AF_CONTROL_SET_PLAYBACK_SPEED;
             }
         }
-        // AF_CONTROL_SET_PLAYBACK_SPEED does not require reinitialization,
-        // while AF_CONTROL_SET_PLAYBACK_SPEED_RESAMPLE requires updating
-        // the samplerate on the resampler, and possibly inserting the
-        // resampler itself.
-        need_reinit |= (method == AF_CONTROL_SET_PLAYBACK_SPEED_RESAMPLE);
     } else {
         if (af_remove_by_label(afs, "playback-speed") < 0)
             return -1;
@@ -107,9 +101,7 @@ static int recreate_audio_filters(struct MPContext *mpctx)
         af_control_any_rev(afs, AF_CONTROL_SET_PLAYBACK_SPEED_RESAMPLE, &speed);
     }
 
-    need_reinit |= afs->initialized < 1;
+    if (afs->initialized < 1 && af_init(afs) < 0) {
-
-    if (need_reinit && af_init(afs) < 0) {
         MP_ERR(mpctx, "Couldn't find matching filter/ao format!\n");
         return -1;
     }

wscript_build.py

@@ -109,7 +109,6 @@ def build(ctx):
         ( "audio/filter/af_equalizer.c" ),
         ( "audio/filter/af_export.c" ),
         ( "audio/filter/af_extrastereo.c" ),
-        ( "audio/filter/af_forcespeed.c" ),
         ( "audio/filter/af_format.c" ),
         ( "audio/filter/af_hrtf.c" ),
         ( "audio/filter/af_karaoke.c" ),