rtpdec: Support sending RTCP feedback packets

This sends NACK for missed packets and PLI (picture loss indication)
if a depacketizer indicates that it needs a new keyframe, according
to RFC 4585.

This is only enabled if the SDP indicated that feedback is supported
(via the AVPF or SAVPF profile names).

The feedback packets are throttled to a certain maximum interval
(currently 250 ms) to make sure the feedback packets don't eat up
too much bandwidth (which might be counterproductive). The RFC
specifies a more elaborate feedback packet scheduling.

The feedback packets are currently sent independently from normal
RTCP RR packets, which is not totally spec compliant, but works
fine in the environments I've tested it in. (RFC 5506 allows this,
but requires a SDP attribute for enabling it.)

Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/rtpdec.c

@@ -30,6 +30,8 @@
 #include "rtpdec.h"
 #include "rtpdec_formats.h"
 
+#define MIN_FEEDBACK_INTERVAL 200000 /* 200 ms in us */
+
 static RTPDynamicProtocolHandler realmedia_mp3_dynamic_handler = {
     .enc_name   = "X-MP3-draft-00",
     .codec_type = AVMEDIA_TYPE_AUDIO,
@@ -366,6 +368,100 @@ void ff_rtp_send_punch_packets(URLContext *rtp_handle)
     av_free(buf);
 }
 
+static int find_missing_packets(RTPDemuxContext *s, uint16_t *first_missing,
+                                uint16_t *missing_mask)
+{
+    int i;
+    uint16_t next_seq = s->seq + 1;
+    RTPPacket *pkt = s->queue;
+
+    if (!pkt || pkt->seq == next_seq)
+        return 0;
+
+    *missing_mask = 0;
+    for (i = 1; i <= 16; i++) {
+        uint16_t missing_seq = next_seq + i;
+        while (pkt) {
+            int16_t diff = pkt->seq - missing_seq;
+            if (diff >= 0)
+                break;
+            pkt = pkt->next;
+        }
+        if (!pkt)
+            break;
+        if (pkt->seq == missing_seq)
+            continue;
+        *missing_mask |= 1 << (i - 1);
+    }
+
+    *first_missing = next_seq;
+    return 1;
+}
+
+int ff_rtp_send_rtcp_feedback(RTPDemuxContext *s, URLContext *fd,
+                              AVIOContext *avio)
+{
+    int len, need_keyframe, missing_packets;
+    AVIOContext *pb;
+    uint8_t *buf;
+    int64_t now;
+    uint16_t first_missing, missing_mask;
+
+    if (!fd && !avio)
+        return -1;
+
+    need_keyframe = s->handler && s->handler->need_keyframe &&
+                    s->handler->need_keyframe(s->dynamic_protocol_context);
+    missing_packets = find_missing_packets(s, &first_missing, &missing_mask);
+
+    if (!need_keyframe && !missing_packets)
+        return 0;
+
+    /* Send new feedback if enough time has elapsed since the last
+     * feedback packet. */
+
+    now = av_gettime();
+    if (s->last_feedback_time &&
+        (now - s->last_feedback_time) < MIN_FEEDBACK_INTERVAL)
+        return 0;
+    s->last_feedback_time = now;
+
+    if (!fd)
+        pb = avio;
+    else if (avio_open_dyn_buf(&pb) < 0)
+        return -1;
+
+    if (need_keyframe) {
+        avio_w8(pb, (RTP_VERSION << 6) | 1); /* PLI */
+        avio_w8(pb, RTCP_PSFB);
+        avio_wb16(pb, 2); /* length in words - 1 */
+        // our own SSRC: we use the server's SSRC + 1 to avoid conflicts
+        avio_wb32(pb, s->ssrc + 1);
+        avio_wb32(pb, s->ssrc); // server SSRC
+    }
+
+    if (missing_packets) {
+        avio_w8(pb, (RTP_VERSION << 6) | 1); /* NACK */
+        avio_w8(pb, RTCP_RTPFB);
+        avio_wb16(pb, 3); /* length in words - 1 */
+        avio_wb32(pb, s->ssrc + 1);
+        avio_wb32(pb, s->ssrc); // server SSRC
+
+        avio_wb16(pb, first_missing);
+        avio_wb16(pb, missing_mask);
+    }
+
+    avio_flush(pb);
+    if (!fd)
+        return 0;
+    len = avio_close_dyn_buf(pb, &buf);
+    if (len > 0 && buf) {
+        ffurl_write(fd, buf, len);
+        av_free(buf);
+    }
+    return 0;
+}
+
 /**
  * open a new RTP parse context for stream 'st'. 'st' can be NULL for
  * MPEG2-TS streams to indicate that they should be demuxed inside the

libavformat/rtpdec.h

@@ -73,6 +73,8 @@ void ff_rtp_send_punch_packets(URLContext* rtp_handle);
  */
 int ff_rtp_check_and_send_back_rr(RTPDemuxContext *s, URLContext *fd,
                                   AVIOContext *avio, int count);
+int ff_rtp_send_rtcp_feedback(RTPDemuxContext *s, URLContext *fd,
+                              AVIOContext *avio);
 
 // these statistics are used for rtcp receiver reports...
 typedef struct RTPStatistics {
@@ -130,6 +132,7 @@ struct RTPDynamicProtocolHandler {
     void (*free)(PayloadContext *protocol_data);
     /** Parse handler for this dynamic packet */
     DynamicPayloadPacketHandlerProc parse_packet;
+    int (*need_keyframe)(PayloadContext *context);
 
     struct RTPDynamicProtocolHandler *next;
 };
@@ -180,6 +183,8 @@ struct RTPDemuxContext {
     unsigned int packet_count;
     unsigned int octet_count;
     unsigned int last_octet_count;
+    int64_t last_feedback_time;
+
     /* buffer for partially parsed packets */
     uint8_t buf[RTP_MAX_PACKET_LENGTH];
 

libavformat/rtsp.c

@@ -379,6 +379,8 @@ static void sdp_parse_line(AVFormatContext *s, SDPParseState *s1,
         get_word(buf1, sizeof(buf1), &p); /* protocol */
         if (!strcmp(buf1, "udp"))
             rt->transport = RTSP_TRANSPORT_RAW;
+        else if (strstr(buf1, "/AVPF") || strstr(buf1, "/SAVPF"))
+            rtsp_st->feedback = 1;
 
         /* XXX: handle list of formats */
         get_word(buf1, sizeof(buf1), &p); /* format list */
@@ -1936,6 +1938,12 @@ redo:
         ret = ff_rdt_parse_packet(rtsp_st->transport_priv, pkt, &rt->recvbuf, len);
     } else if (rt->transport == RTSP_TRANSPORT_RTP) {
         ret = ff_rtp_parse_packet(rtsp_st->transport_priv, pkt, &rt->recvbuf, len);
+        if (rtsp_st->feedback) {
+            AVIOContext *pb = NULL;
+            if (rt->lower_transport == RTSP_LOWER_TRANSPORT_CUSTOM)
+                pb = s->pb;
+            ff_rtp_send_rtcp_feedback(rtsp_st->transport_priv, rtsp_st->rtp_handle, pb);
+        }
         if (ret < 0) {
             /* Either bad packet, or a RTCP packet. Check if the
              * first_rtcp_ntp_time field was initialized. */

libavformat/rtsp.h

@@ -437,6 +437,9 @@ typedef struct RTSPStream {
     /** private data associated with the dynamic protocol */
     PayloadContext *dynamic_protocol_context;
     //@}
+
+    /** Enable sending RTCP feedback messages according to RFC 4585 */
+    int feedback;
 } RTSPStream;
 
 void ff_rtsp_parse_line(RTSPMessageHeader *reply, const char *buf,

libavformat/version.h

@@ -31,7 +31,7 @@
 
 #define LIBAVFORMAT_VERSION_MAJOR 54
 #define LIBAVFORMAT_VERSION_MINOR 20
-#define LIBAVFORMAT_VERSION_MICRO  3
+#define LIBAVFORMAT_VERSION_MICRO  4
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
                                                LIBAVFORMAT_VERSION_MINOR, \