Handle audio stream discontinuities

The audio regulator assumed a continuous audio stream. But some audio
sources (like the "voice call" audio source) do not produce any packets
on silence, breaking this assumption.

Use PTS to detect such discontinuities.

PR #5870 <https://github.com/Genymobile/scrcpy/pull/5870>

app/src/audio_regulator.c

@@ -141,6 +141,36 @@ bool
 sc_audio_regulator_push(struct sc_audio_regulator *ar, const AVFrame *frame) {
     SwrContext *swr_ctx = ar->swr_ctx;
 
+    uint32_t input_samples = frame->nb_samples;
+
+    assert(frame->pts >= 0);
+    int64_t pts = frame->pts;
+    if (ar->next_expected_pts && pts - ar->next_expected_pts > 100000) {
+        LOGV("[Audio] Discontinuity detected: %" PRIi64 "µs",
+             pts - ar->next_expected_pts);
+        // More than 100ms: consider it as a discontinuity
+        // (typically because silence packets were not captured)
+        uint32_t can_read = sc_audiobuf_can_read(&ar->buf);
+        if (input_samples + can_read < ar->target_buffering) {
+            // Adjust buffering to the target value directly
+            uint32_t silence = ar->target_buffering - can_read - input_samples;
+            sc_audiobuf_write_silence(&ar->buf, silence);
+        }
+
+        // Reset state
+        ar->avg_buffering.avg = ar->target_buffering;
+        int ret = swr_set_compensation(swr_ctx, 0, 0);
+        (void) ret;
+        assert(!ret); // disabling compensation should never fail
+        ar->compensation_active = false;
+        ar->samples_since_resync = 0;
+        atomic_store_explicit(&ar->underflow, 0, memory_order_relaxed);
+    }
+
+    int64_t packet_duration = input_samples * INT64_C(1000000)
+                            / ar->sample_rate;
+    ar->next_expected_pts = pts + packet_duration;
+
     int64_t swr_delay = swr_get_delay(swr_ctx, ar->sample_rate);
     // No need to av_rescale_rnd(), input and output sample rates are the same.
     // Add more space (256) for clock compensation.
@@ -260,7 +290,7 @@ sc_audio_regulator_push(struct sc_audio_regulator *ar, const AVFrame *frame) {
     }
 
     // Number of samples added (or removed, if negative) for compensation
-    int32_t instant_compensation = (int32_t) written - frame->nb_samples;
+    int32_t instant_compensation = (int32_t) written - input_samples;
     // Inserting silence instantly increases buffering
     int32_t inserted_silence = (int32_t) underflow;
     // Dropping input samples instantly decreases buffering
@@ -403,6 +433,7 @@ sc_audio_regulator_init(struct sc_audio_regulator *ar, size_t sample_size,
     atomic_init(&ar->underflow, 0);
     ar->underflow_report = 0;
     ar->compensation_active = false;
+    ar->next_expected_pts = 0;
 
     return true;
 

app/src/audio_regulator.h

@@ -57,6 +57,9 @@ struct sc_audio_regulator {
 
     // Set to true the first time samples are pulled by the player
     atomic_bool played;
+
+    // PTS of the next expected packet (useful to detect discontinuities)
+    int64_t next_expected_pts;
 };
 
 bool

app/src/util/audiobuf.c

@@ -116,3 +116,38 @@ sc_audiobuf_write(struct sc_audiobuf *buf, const void *from_,
 
     return samples_count;
 }
+
+uint32_t
+sc_audiobuf_write_silence(struct sc_audiobuf *buf, uint32_t samples_count) {
+    // Only the writer thread can write head, so memory_order_relaxed is
+    // sufficient
+    uint32_t head = atomic_load_explicit(&buf->head, memory_order_relaxed);
+
+    // The tail cursor is updated after the data is consumed by the reader
+    uint32_t tail = atomic_load_explicit(&buf->tail, memory_order_acquire);
+
+    uint32_t can_write = (buf->alloc_size + tail - head - 1) % buf->alloc_size;
+    if (!can_write) {
+        return 0;
+    }
+    if (samples_count > can_write) {
+        samples_count = can_write;
+    }
+
+    uint32_t right_count = buf->alloc_size - head;
+    if (right_count > samples_count) {
+        right_count = samples_count;
+    }
+    memset(buf->data + (head * buf->sample_size), 0,
+           right_count * buf->sample_size);
+
+    if (samples_count > right_count) {
+        uint32_t left_count = samples_count - right_count;
+        memset(buf->data, 0, left_count * buf->sample_size);
+    }
+
+    uint32_t new_head = (head + samples_count) % buf->alloc_size;
+    atomic_store_explicit(&buf->head, new_head, memory_order_release);
+
+    return samples_count;
+}

app/src/util/audiobuf.h

@@ -50,6 +50,9 @@ uint32_t
 sc_audiobuf_write(struct sc_audiobuf *buf, const void *from,
                   uint32_t samples_count);
 
+uint32_t
+sc_audiobuf_write_silence(struct sc_audiobuf *buf, uint32_t samples);
+
 static inline uint32_t
 sc_audiobuf_capacity(struct sc_audiobuf *buf) {
     assert(buf->alloc_size);

app/tests/test_audiobuf.c

@@ -113,6 +113,14 @@ static void test_audiobuf_partial_read_write(void) {
     uint32_t expected2[] = {4, 5, 6, 1, 2, 3, 4, 1, 2, 3};
     assert(!memcmp(data, expected2, 12));
 
+    w = sc_audiobuf_write_silence(&buf, 4);
+    assert(w == 4);
+
+    r = sc_audiobuf_read(&buf, data, 4);
+    assert(r == 4);
+    uint32_t expected3[] = {0, 0, 0, 0};
+    assert(!memcmp(data, expected3, 4));
+
     sc_audiobuf_destroy(&buf);
 }