diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index e4e1875bd4..7de129b5c5 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3567,20 +3567,20 @@ static int execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count){
                                              "in complementary field pair "
                                              "(first field is long term)\n");
         } else {
-        pic= remove_short(h, s->current_picture_ptr->frame_num);
-        if(pic){
-            unreference_pic(h, pic, 0);
-            av_log(h->s.avctx, AV_LOG_ERROR, "illegal short term buffer state detected\n");
+            pic= remove_short(h, s->current_picture_ptr->frame_num);
+            if(pic){
+                unreference_pic(h, pic, 0);
+                av_log(h->s.avctx, AV_LOG_ERROR, "illegal short term buffer state detected\n");
+            }
+
+            if(h->short_ref_count)
+                memmove(&h->short_ref[1], &h->short_ref[0], h->short_ref_count*sizeof(Picture*));
+
+            h->short_ref[0]= s->current_picture_ptr;
+            h->short_ref[0]->long_ref=0;
+            h->short_ref_count++;
+            s->current_picture_ptr->reference |= s->picture_structure;
         }
-
-        if(h->short_ref_count)
-            memmove(&h->short_ref[1], &h->short_ref[0], h->short_ref_count*sizeof(Picture*));
-
-        h->short_ref[0]= s->current_picture_ptr;
-        h->short_ref[0]->long_ref=0;
-        h->short_ref_count++;
-        s->current_picture_ptr->reference |= s->picture_structure;
-    }
     }
 
     if (h->long_ref_count + h->short_ref_count > h->sps.ref_frame_count){