diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c
index 8a936da59e..d9f60d6d23 100644
--- a/libavcodec/aacdec.c
+++ b/libavcodec/aacdec.c
@@ -570,9 +570,6 @@ static av_cold int aac_decode_init(AVCodecContext *avctx)
     ac->m4ac.sample_rate = avctx->sample_rate;
 
     if (avctx->extradata_size > 0) {
-        avctx->channels    = 0;
-        avctx->frame_size  = 0;
-        avctx->sample_rate = 0;
         if (decode_audio_specific_config(ac, ac->avctx, &ac->m4ac,
                                          avctx->extradata,
                                          avctx->extradata_size) < 0)
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 654bc8aae5..b9a4ebab9d 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -2261,6 +2261,15 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options)
     for(i=0;i<ic->nb_streams;i++) {
         AVCodec *codec;
         st = ic->streams[i];
+        if (st->codec->codec_id == CODEC_ID_AAC && st->codec->extradata_size) {
+            // We need to discard these since they can be plain wrong for
+            // backwards compatible HE-AAC signaling.
+            // But when we have no extradata we need to keep them or we can't
+            // play anything at all.
+            st->codec->sample_rate = 0;
+            st->codec->frame_size = 0;
+            st->codec->channels = 0;
+        }
 
         if (st->codec->codec_type == AVMEDIA_TYPE_VIDEO ||
             st->codec->codec_type == AVMEDIA_TYPE_SUBTITLE) {