diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
index 1e6bd6eee7..a5ec1caf7a 100644
--- a/libavformat/rtpdec.c
+++ b/libavformat/rtpdec.c
@@ -421,7 +421,10 @@ static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestam
 {
     if (pkt->pts != AV_NOPTS_VALUE || pkt->dts != AV_NOPTS_VALUE)
         return; /* Timestamp already set by depacketizer */
-    if (s->last_rtcp_ntp_time != AV_NOPTS_VALUE && timestamp != RTP_NOTS_VALUE) {
+    if (timestamp == RTP_NOTS_VALUE)
+        return;
+
+    if (s->last_rtcp_ntp_time != AV_NOPTS_VALUE) {
         int64_t addend;
         int delta_timestamp;
 
@@ -433,8 +436,7 @@ static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestam
                    delta_timestamp;
         return;
     }
-    if (timestamp == RTP_NOTS_VALUE)
-        return;
+
     if (!s->base_timestamp)
         s->base_timestamp = timestamp;
     pkt->pts = s->range_start_offset + timestamp - s->base_timestamp;