diff --git a/player/core.h b/player/core.h
index a4ffc34b68..0dbf56d2b1 100644
--- a/player/core.h
+++ b/player/core.h
@@ -428,6 +428,7 @@ typedef struct MPContext {
     bool demux_underrun;
     double cache_stop_time;
     int cache_buffer;
+    double cache_update_pts;
 
     // Set after showing warning about decoding being too slow for realtime
     // playback rate. Used to avoid showing it multiple times.
diff --git a/player/playloop.c b/player/playloop.c
index 6fcd986ebd..dd5f016f1e 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -244,6 +244,7 @@ void reset_playback_state(struct MPContext *mpctx)
     mpctx->restart_complete = false;
     mpctx->paused_for_cache = false;
     mpctx->cache_buffer = 100;
+    mpctx->cache_update_pts = MP_NOPTS_VALUE;
 
     encode_lavc_discontinuity(mpctx->encode_lavc_ctx);
 
@@ -751,6 +752,8 @@ static void handle_update_cache(struct MPContext *mpctx)
 
     // Also update cache properties.
     bool busy = !s.idle;
+    if (fabs(mpctx->cache_update_pts - mpctx->playback_pts) >= 1.0)
+        busy = true;
     if (busy || mpctx->next_cache_update > 0) {
         if (mpctx->next_cache_update <= now) {
             mpctx->next_cache_update = busy ? now + 0.25 : 0;
@@ -781,8 +784,10 @@ static void handle_update_cache(struct MPContext *mpctx)
     if (s.eof && !busy)
         prefetch_next(mpctx);
 
-    if (force_update)
+    if (force_update) {
+        mpctx->cache_update_pts = mpctx->playback_pts;
         mp_notify(mpctx, MP_EVENT_CACHE_UPDATE, NULL);
+    }
 }
 
 int get_cache_buffering_percentage(struct MPContext *mpctx)