diff --git a/libavcodec/mlpdec.c b/libavcodec/mlpdec.c
index 68c270ef52..780ed5b7b6 100644
--- a/libavcodec/mlpdec.c
+++ b/libavcodec/mlpdec.c
@@ -417,7 +417,8 @@ static int read_major_sync(MLPDecodeContext *m, GetBitContext *gb)
                 m->substream[2].mask = mh.channel_layout_thd_stream2;
             else
                 m->substream[2].mask = mh.channel_layout_thd_stream1;
-        m->substream[1].mask = mh.channel_layout_thd_stream1;
+        if (m->avctx->channels > 2)
+            m->substream[mh.num_substreams > 1].mask = mh.channel_layout_thd_stream1;
 
         if (m->avctx->channels<=2 && m->substream[substr].mask == AV_CH_LAYOUT_MONO && m->max_decoded_substream == 1) {
             av_log(m->avctx, AV_LOG_DEBUG, "Mono stream with 2 substreams, ignoring 2nd\n");