diff --git a/libavcodec/opusenc.c b/libavcodec/opusenc.c
index 578785f4b4..3c08ebcf69 100644
--- a/libavcodec/opusenc.c
+++ b/libavcodec/opusenc.c
@@ -543,7 +543,7 @@ static int opus_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
         ff_bufqueue_add(avctx, &s->bufqueue, av_frame_clone(frame));
     } else {
         ff_opus_psy_signal_eof(&s->psyctx);
-        if (!s->afq.remaining_samples)
+        if (!s->afq.remaining_samples || !avctx->frame_number)
             return 0; /* We've been flushed and there's nothing left to encode */
     }