diff --git a/stream/cache.c b/stream/cache.c
index bcb5bea086..c992764522 100644
--- a/stream/cache.c
+++ b/stream/cache.c
@@ -252,10 +252,15 @@ static bool cache_fill(struct priv *s)
                    "cached range: %"PRId64"-%"PRId64".\n", read,
                    s->min_filepos, s->max_filepos);
             cache_drop_contents(s);
-            stream_seek(s->stream, read);
         }
     }
 
+    if (stream_tell(s->stream) != s->max_filepos) {
+        MP_VERBOSE(s, "Seeking underlying stream: %"PRId64" -> %"PRId64"\n",
+                   stream_tell(s->stream), s->max_filepos);
+        stream_seek(s->stream, s->max_filepos);
+    }
+
     // number of buffer bytes which should be preserved in backwards direction
     int64_t back = mp_clipi64(read - s->min_filepos, 0, s->back_size);