diff --git a/player/audio.c b/player/audio.c
index 20f80256b2..eb6081e26d 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -181,6 +181,7 @@ static void ao_chain_reset_state(struct ao_chain *ao_c)
     ao_c->last_out_pts = MP_NOPTS_VALUE;
     TA_FREEP(&ao_c->output_frame);
     ao_c->out_eof = false;
+    ao_c->underrun = false;
 
     mp_audio_buffer_clear(ao_c->ao_buffer);
 }
@@ -855,7 +856,14 @@ void fill_audio_out_buffers(struct MPContext *mpctx)
 
     int playsize = ao_get_space(mpctx->ao);
 
-    ao_query_and_reset_events(mpctx->ao, AO_EVENT_UNDERRUN);
+    if (ao_query_and_reset_events(mpctx->ao, AO_EVENT_UNDERRUN))
+        ao_c->underrun = true;
+
+    // Stop feeding data if an underrun happened. Something else needs to
+    // "unblock" audio after underrun. handle_update_cache() does this and can
+    // take the network state into account.
+    if (ao_c->underrun)
+        return;
 
     int skip = 0;
     bool sync_known = get_sync_samples(mpctx, &skip);
diff --git a/player/core.h b/player/core.h
index 11e741bb78..0edf4fa31e 100644
--- a/player/core.h
+++ b/player/core.h
@@ -183,6 +183,8 @@ struct vo_chain {
     bool is_coverart;
     // - video consists of sparse still images
     bool is_sparse;
+
+    bool underrun;
 };
 
 // Like vo_chain, for audio.
@@ -206,6 +208,8 @@ struct ao_chain {
     struct track *track;
     struct mp_pin *filter_src;
     struct mp_pin *dec_src;
+
+    bool underrun;
 };
 
 /* Note that playback can be paused, stopped, etc. at any time. While paused,
@@ -423,6 +427,7 @@ typedef struct MPContext {
     bool playing_msg_shown;
 
     bool paused_for_cache;
+    bool demux_underrun;
     double cache_stop_time;
     int cache_buffer;
 
diff --git a/player/playloop.c b/player/playloop.c
index e0d0056d7d..99069401de 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -660,18 +660,36 @@ static void handle_osd_redraw(struct MPContext *mpctx)
     vo_redraw(mpctx->video_out);
 }
 
+static void clear_underruns(struct MPContext *mpctx)
+{
+    if (mpctx->ao_chain && mpctx->ao_chain->underrun) {
+        mpctx->ao_chain->underrun = false;
+        mp_wakeup_core(mpctx);
+    }
+
+    if (mpctx->vo_chain && mpctx->vo_chain->underrun) {
+        mpctx->vo_chain->underrun = false;
+        mp_wakeup_core(mpctx);
+    }
+}
+
 static void handle_update_cache(struct MPContext *mpctx)
 {
     bool force_update = false;
     struct MPOpts *opts = mpctx->opts;
-    if (!mpctx->demuxer)
+
+    if (!mpctx->demuxer) {
+        clear_underruns(mpctx);
         return;
+    }
 
     double now = mp_time_sec();
 
     struct demux_reader_state s;
     demux_get_reader_state(mpctx->demuxer, &s);
 
+    mpctx->demux_underrun |= s.underrun;
+
     int cache_buffer = 100;
     bool use_pause_on_low_cache = demux_is_network_cached(mpctx->demuxer) &&
                                   opts->cache_pause && mpctx->play_dir > 0;
@@ -690,17 +708,43 @@ static void handle_update_cache(struct MPContext *mpctx)
 
     // Enter buffering state only if there actually was an underrun (or if
     // initial caching before playback restart is used).
-    if (is_low && !mpctx->paused_for_cache && mpctx->restart_complete)
-        is_low = s.underrun;
+    bool need_wait = is_low;
+    if (is_low && !mpctx->paused_for_cache && mpctx->restart_complete) {
+        // Wait only if an output underrun was registered. (Or if there is no
+        // underrun detection.)
+        bool output_underrun = false;
 
-    if (mpctx->paused_for_cache != is_low) {
-        mpctx->paused_for_cache = is_low;
+        if (mpctx->ao_chain) {
+            output_underrun |=
+                !(mpctx->ao && ao_get_reports_underruns(mpctx->ao)) ||
+                mpctx->ao_chain->underrun;
+        }
+        if (mpctx->vo_chain)
+            output_underrun |= mpctx->vo_chain->underrun;
+
+        // Output underruns could be sporadic (unrelated to demuxer buffer state
+        // and for example caused by slow decoding), so use a past demuxer
+        // underrun as indication that the underrun was possibly due to a
+        // demuxer underrun.
+        need_wait = mpctx->demux_underrun && output_underrun;
+    }
+
+    // Let the underrun flag "stick" around until the cache has fully recovered.
+    // See logic where demux_underrun is used.
+    if (!is_low)
+        mpctx->demux_underrun = false;
+
+    if (mpctx->paused_for_cache != need_wait) {
+        mpctx->paused_for_cache = need_wait;
         update_internal_pause_state(mpctx);
         force_update = true;
-        if (is_low)
+        if (mpctx->paused_for_cache)
             mpctx->cache_stop_time = now;
     }
 
+    if (!mpctx->paused_for_cache)
+        clear_underruns(mpctx);
+
     if (mpctx->paused_for_cache) {
         cache_buffer =
             100 * MPCLAMP(s.ts_duration / opts->cache_pause_wait, 0, 0.99);
diff --git a/player/video.c b/player/video.c
index 1d800a96d3..680636b075 100644
--- a/player/video.c
+++ b/player/video.c
@@ -91,6 +91,7 @@ int reinit_video_filters(struct MPContext *mpctx)
 static void vo_chain_reset_state(struct vo_chain *vo_c)
 {
     vo_seek_reset(vo_c->vo);
+    vo_c->underrun = false;
 }
 
 void reset_video_state(struct MPContext *mpctx)
@@ -1003,8 +1004,13 @@ void write_video(struct MPContext *mpctx)
     if (r < 0)
         goto error;
 
-    if (r == VD_WAIT) // Demuxer will wake us up for more packets to decode.
+    if (r == VD_WAIT) {
+        // Heuristic to detect underruns.
+        if (mpctx->video_status == STATUS_PLAYING && !vo_still_displaying(vo))
+            vo_c->underrun = true;
+        // Demuxer will wake us up for more packets to decode.
         return;
+    }
 
     if (r == VD_EOF) {
         if (check_for_hwdec_fallback(mpctx))