diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c
index 0a7c39eacd..1667e9f08b 100644
--- a/libavformat/aviobuf.c
+++ b/libavformat/aviobuf.c
@@ -542,7 +542,7 @@ static void fill_buffer(AVIOContext *s)
 
     /* make buffer smaller in case it ended up large after probing */
     if (s->read_packet && s->orig_buffer_size && s->buffer_size > s->orig_buffer_size) {
-        if (dst == s->buffer) {
+        if (dst == s->buffer && s->buf_ptr != dst) {
             int ret = ffio_set_buf_size(s, s->orig_buffer_size);
             if (ret < 0)
                 av_log(s, AV_LOG_WARNING, "Failed to decrease buffer size\n");