diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index 0cf0d46022..f98389868d 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -49,7 +49,8 @@ static int get_scale_factor(H264Context *const h, int poc, int poc1, int i)
 
 void ff_h264_direct_dist_scale_factor(H264Context *const h)
 {
-    const int poc  = h->cur_pic_ptr->field_poc[h->picture_structure == PICT_BOTTOM_FIELD];
+    const int poc  = FIELD_PICTURE(h) ? h->cur_pic_ptr->field_poc[h->picture_structure == PICT_BOTTOM_FIELD]
+                                      : h->cur_pic_ptr->poc;
     const int poc1 = h->ref_list[1][0].poc;
     int i, field;