diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index 5ba07448a6..28d3cbc2c3 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -430,7 +430,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
 
     sps->ref_frame_count= get_ue_golomb_31(&s->gb);
     if (h->s.avctx->codec_tag == MKTAG('S', 'M', 'V', '2'))
-        sps->ref_frame_count= 2;
+        sps->ref_frame_count= FFMAX(2, sps->ref_frame_count);
     if(sps->ref_frame_count > MAX_PICTURE_COUNT-2 || sps->ref_frame_count > 16U){
         av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n");
         goto fail;