diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
index 71c0c34681..43b3505f3c 100644
--- a/libavformat/hlsenc.c
+++ b/libavformat/hlsenc.c
@@ -253,10 +253,10 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
         hls->start_pts = pkt->pts;
         hls->end_pts   = pkt->pts;
     }
-    end_pts += hls->start_pts;
 
     if ((hls->has_video && st->codec->codec_type == AVMEDIA_TYPE_VIDEO)      &&
-        av_compare_ts(pkt->pts, st->time_base, end_pts, AV_TIME_BASE_Q) >= 0 &&
+        av_compare_ts(pkt->pts - hls->start_pts, st->time_base,
+                      end_pts, AV_TIME_BASE_Q) >= 0 &&
         pkt->flags & AV_PKT_FLAG_KEY) {
 
         ret = append_entry(hls, av_rescale(pkt->pts - hls->end_pts,