diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 3d7540792e..313b45ad97 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -1831,7 +1831,7 @@ int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
             av_log(avctx, AV_LOG_DEBUG, "skip %d samples due to side data\n",
                    avctx->internal->skip_samples);
         }
-        if (avctx->internal->skip_samples) {
+        if (avctx->internal->skip_samples && *got_frame_ptr) {
             if(frame->nb_samples <= avctx->internal->skip_samples){
                 *got_frame_ptr = 0;
                 avctx->internal->skip_samples -= frame->nb_samples;