diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
index 01822a8b4d..e9b8a46b5a 100644
--- a/libavformat/rtpdec.c
+++ b/libavformat/rtpdec.c
@@ -400,7 +400,11 @@ static int rtp_parse_mp4_au(RTPDemuxContext *s, const uint8_t *buf)
         return -1;
 
     infos->nb_au_headers = au_headers_length / au_header_size;
+    if (!infos->au_headers || infos->au_headers_allocated < infos->nb_au_headers) {
+        av_free(infos->au_headers);
     infos->au_headers = av_malloc(sizeof(struct AUHeaders) * infos->nb_au_headers);
+        infos->au_headers_allocated = infos->nb_au_headers;
+    }
 
     /* XXX: We handle multiple AU Section as only one (need to fix this for interleaving)
        In my test, the FAAD decoder does not behave correctly when sending each AU one by one
@@ -599,6 +603,8 @@ int rtp_parse_packet(RTPDemuxContext *s, AVPacket *pkt,
 void rtp_parse_close(RTPDemuxContext *s)
 {
     // TODO: fold this into the protocol specific data fields.
+    av_free(s->rtp_payload_data->mode);
+    av_free(s->rtp_payload_data->au_headers);
     if (!strcmp(ff_rtp_enc_name(s->payload_type), "MP2T")) {
         ff_mpegts_parse_close(s->ts);
     }
diff --git a/libavformat/rtpdec.h b/libavformat/rtpdec.h
index 1e56e9e40a..19996fadd4 100644
--- a/libavformat/rtpdec.h
+++ b/libavformat/rtpdec.h
@@ -48,6 +48,7 @@ typedef struct rtp_payload_data
         int rap_flag;
         int streamstate;
     } *au_headers;
+    int au_headers_allocated;
     int nb_au_headers;
     int au_headers_length_bytes;
     int cur_au_index;