player: add audio drop/duplicate mode

Not very robust in the moment.

DOCS/man/options.rst

@@ -3463,6 +3463,11 @@ Miscellaneous
     :display-vdrop:     Drop or repeat video frames to compensate desyncing
                         video. (Although it should have the same effects as
                         ``audio``, the implementation is very different.)
+    :display-adrop:     Drop or repeat audio data to compensate desyncing
+                        video. See ``--video-sync-adrop-size``. This mode will
+                        cause severe audio artifacts if the real monitor
+                        refresh rate is too different from the reported or
+                        forced rate.
     :display-desync:    Sync video to display, and let audio play on its own.
     :desync:            Sync video according to system clock, and let audio play
                         on its own.
@@ -3494,6 +3499,13 @@ Miscellaneous
     frame dropping due to the audio "overshooting" and skipping multiple video
     frames before the sync logic can react.
 
+``--video-sync-adrop-size=<value``
+    For the ``--video-sync=display-adrop`` mode. This mode duplicates/drops
+    audio data to keep audio in sync with video. To avoid audio artifacts on
+    jitter (which would add/remove samples all the time), this is done in
+    relatively large, fixed units, controlled by this option. The unit is
+    seconds.
+
 ``--mf-fps=<value>``
     Framerate used when decoding from multiple PNG or JPEG files with ``mf://``
     (default: 1).

audio/audio_buffer.c

@@ -119,6 +119,15 @@ void mp_audio_buffer_prepend_silence(struct mp_audio_buffer *ab, int samples)
     mp_audio_fill_silence(ab->buffer, 0, samples);
 }
 
+void mp_audio_buffer_duplicate(struct mp_audio_buffer *ab, int samples)
+{
+    assert(samples >= 0 && samples <= ab->buffer->samples);
+    int oldlen = ab->buffer->samples;
+    ab->buffer->samples += samples;
+    mp_audio_realloc_min(ab->buffer, ab->buffer->samples);
+    mp_audio_copy(ab->buffer, oldlen, ab->buffer, oldlen - samples, samples);
+}
+
 // Get the start of the current readable buffer.
 void mp_audio_buffer_peek(struct mp_audio_buffer *ab, struct mp_audio *out_mpa)
 {

audio/audio_buffer.h

@@ -35,6 +35,7 @@ void mp_audio_buffer_get_write_buffer(struct mp_audio_buffer *ab, int minsamples
 void mp_audio_buffer_finish_write(struct mp_audio_buffer *ab, int samples);
 void mp_audio_buffer_append(struct mp_audio_buffer *ab, struct mp_audio *mpa);
 void mp_audio_buffer_prepend_silence(struct mp_audio_buffer *ab, int samples);
+void mp_audio_buffer_duplicate(struct mp_audio_buffer *ab, int samples);
 void mp_audio_buffer_peek(struct mp_audio_buffer *ab, struct mp_audio *out_mpa);
 void mp_audio_buffer_skip(struct mp_audio_buffer *ab, int samples);
 void mp_audio_buffer_clear(struct mp_audio_buffer *ab);

options/options.c

@@ -528,6 +528,7 @@ const m_option_t mp_opts[] = {
                 {"display-resample", VS_DISP_RESAMPLE},
                 {"display-resample-vdrop", VS_DISP_RESAMPLE_VDROP},
                 {"display-resample-desync", VS_DISP_RESAMPLE_NONE},
+                {"display-adrop", VS_DISP_ADROP},
                 {"display-vdrop", VS_DISP_VDROP},
                 {"display-desync", VS_DISP_NONE},
                 {"desync", VS_NONE})),
@@ -535,6 +536,8 @@ const m_option_t mp_opts[] = {
                M_OPT_MIN, .min = 0),
     OPT_DOUBLE("video-sync-max-audio-change", sync_max_audio_change,
                M_OPT_MIN | M_OPT_MAX, .min = 0, .max = 1),
+    OPT_DOUBLE("video-sync-adrop-size", sync_audio_drop_size,
+               M_OPT_MIN | M_OPT_MAX, .min = 0, .max = 1),
     OPT_CHOICE("hr-seek", hr_seek, 0,
                ({"no", -1}, {"absolute", 0}, {"yes", 1}, {"always", 1})),
     OPT_FLOAT("hr-seek-demuxer-offset", hr_seek_demuxer_offset, 0),
@@ -728,6 +731,7 @@ const struct MPOpts mp_default_opts = {
     .hr_seek_framedrop = 1,
     .sync_max_video_change = 1,
     .sync_max_audio_change = 0.125,
+    .sync_audio_drop_size = 0.020,
     .load_config = 1,
     .position_resume = 1,
     .stream_cache = {

options/options.h

@@ -148,6 +148,7 @@ typedef struct MPOpts {
     int video_sync;
     double sync_max_video_change;
     double sync_max_audio_change;
+    double sync_audio_drop_size;
     int hr_seek;
     float hr_seek_demuxer_offset;
     int hr_seek_framedrop;

player/audio.c

@@ -551,6 +551,27 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         playsize = MPMAX(1, playsize + skip); // silence will be prepended
     }
 
+    int skip_duplicate = 0; // >0: skip, <0: duplicate
+    if (mpctx->display_sync_active && opts->video_sync == VS_DISP_ADROP &&
+        fabs(mpctx->last_av_difference) >= opts->sync_audio_drop_size &&
+        mpctx->audio_status == STATUS_PLAYING)
+    {
+        int samples = ceil(opts->sync_audio_drop_size * play_samplerate);
+        int align = af_format_sample_alignment(out_format.format);
+        samples = (samples + align / 2) / align * align;
+
+        skip_duplicate = mpctx->last_av_difference >= 0 ? -samples : samples;
+
+        // safeguard against excessively growing audio buffer sizes
+        if (skip_duplicate < 0 && mp_audio_buffer_seconds(mpctx->ao_buffer) > 5) {
+            skip_duplicate = 0;
+            samples = 0;
+        }
+
+        if (playsize < samples)
+            playsize = samples;
+    }
+
     int status = AD_OK;
     bool working = false;
     if (playsize > mp_audio_buffer_samples(mpctx->ao_buffer)) {
@@ -598,6 +619,21 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         end_sync = true;
     }
 
+    if (skip_duplicate) {
+        int max = mp_audio_buffer_samples(mpctx->ao_buffer);
+        if (abs(skip_duplicate) > max)
+            skip_duplicate = skip_duplicate >= 0 ? max : -max;
+        mpctx->last_av_difference += skip_duplicate / play_samplerate;
+        if (skip_duplicate >= 0) {
+            mp_audio_buffer_skip(mpctx->ao_buffer, skip_duplicate);
+            MP_STATS(mpctx, "drop-audio");
+        } else {
+            mp_audio_buffer_duplicate(mpctx->ao_buffer, -skip_duplicate);
+            MP_STATS(mpctx, "duplicate-audio");
+        }
+        MP_VERBOSE(mpctx, "audio skip_duplicate=%d\n", skip_duplicate);
+    }
+
     if (mpctx->audio_status == STATUS_SYNCING) {
         if (end_sync)
             mpctx->audio_status = STATUS_FILLING;

player/core.h

@@ -84,6 +84,7 @@ enum video_sync {
     VS_DISP_RESAMPLE,
     VS_DISP_RESAMPLE_VDROP,
     VS_DISP_RESAMPLE_NONE,
+    VS_DISP_ADROP,
     VS_DISP_VDROP,
     VS_DISP_NONE,
     VS_NONE,
@@ -92,6 +93,7 @@ enum video_sync {
 #define VS_IS_DISP(x) ((x) == VS_DISP_RESAMPLE ||       \
                        (x) == VS_DISP_RESAMPLE_VDROP || \
                        (x) == VS_DISP_RESAMPLE_NONE ||  \
+                       (x) == VS_DISP_ADROP ||          \
                        (x) == VS_DISP_VDROP ||          \
                        (x) == VS_DISP_NONE)
 

player/video.c

@@ -912,7 +912,7 @@ static void handle_display_sync_frame(struct MPContext *mpctx,
     bool resample = mode == VS_DISP_RESAMPLE || mode == VS_DISP_RESAMPLE_VDROP ||
                     mode == VS_DISP_RESAMPLE_NONE;
     bool drop = mode == VS_DISP_VDROP || mode == VS_DISP_RESAMPLE ||
-                mode == VS_DISP_RESAMPLE_VDROP;
+                mode == VS_DISP_ADROP || mode == VS_DISP_RESAMPLE_VDROP;
     drop &= (opts->frame_dropping & 1);
 
     if (resample && using_spdif_passthrough(mpctx))