demux_mkv: support FFmpeg A_MS/ACM extensions

Indeed, FFmpeg found a way to maximize the misery around VfW/AVI-style
muxing. It appears it can mux a number of random codecs by using random
format tags. To make this even more stranger, it has a probably custom
GUID for signaling them, although for unknown reasons this is done only
"sometimes" (judging from FFmpeg's riffenc.c).

Whatever, it's not too hard to support it. Also apparently fix the
incorrect interpretation of extended formats - there's absolutely no
reason to assume they're always PCM. Instead, check for the correct
GUIDs. Also while we're at it, move the channel mask handling also to
codec_tag.c, so all WAVEFORMATEXTENSIBLE handling is in one place. (With
the normal wav header handling strangely still in demux_mkv.c.)

The case I was looking at (aac_latm muxing) decodes now. While I'm not
entirely sure about its correctness (libavformat has a weird
special-case for SBR), it certainly doesn't try to play it as PCM,
which is much of an improvement.

The extradata mess in the demux_mkv.c A_MS/ACM code path is unfortunate
and ugly, but has less impact than refactoring all the code to make
this specific case nicer.

Did I mention yet that I hate VfW-style mkv muxing?

demux/codec_tags.c

@@ -18,6 +18,8 @@
 #include <libavformat/avformat.h>
 #include <libavcodec/avcodec.h>
 #include <libavutil/common.h>
+#include <libavutil/intreadwrite.h>
+
 #include "codec_tags.h"
 #include "stheader.h"
 #include "common/av_common.h"
@@ -48,14 +50,50 @@ static const char *lookup_tag(int type, uint32_t tag)
     return id == AV_CODEC_ID_NONE ? NULL : mp_codec_from_av_codec_id(id);
 }
 
+static const unsigned char guid_pcm[16] =
+    {0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
+     0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71};
+static const unsigned char guid_float[16] =
+    {0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
+     0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71};
+// Corresponds to FF_MEDIASUBTYPE_BASE_GUID (plus 4 bytes of padding).
+static const unsigned char guid_ffext[16] =
+    {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00,
+     0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71};
+
 static void map_audio_pcm_tag(struct mp_codec_params *c)
 {
+    // MS PCM, Extended
+    if (c->codec_tag == 0xfffe && c->extradata_size >= 22) {
+        // WAVEFORMATEXTENSIBLE.dwChannelMask
+        uint64_t chmask = AV_RL32(c->extradata + 2);
+        struct mp_chmap chmap;
+        mp_chmap_from_waveext(&chmap, chmask);
+        if (c->channels.num == chmap.num)
+            c->channels = chmap;
+
+        // WAVEFORMATEXTENSIBLE.SubFormat
+        unsigned char *subformat = c->extradata + 6;
+        if (memcmp(subformat + 4, guid_ffext + 4, 12) == 0) {
+            // libavformat extension.
+            c->codec_tag = AV_RL32(subformat);
+            c->codec = lookup_tag(c->type, c->codec_tag);
+        }
+        if (memcmp(subformat, guid_pcm, 16) == 0)
+            c->codec_tag = 0x0;
+        if (memcmp(subformat, guid_float, 16) == 0)
+            c->codec_tag = 0x3;
+
+        // Compressed formats might use this.
+        c->extradata += 22;
+        c->extradata_size += 22;
+    }
+
     int bits = c->bits_per_coded_sample;
     int bytes = (bits + 7) / 8;
     switch (c->codec_tag) {
     case 0x0:       // Microsoft PCM
     case 0x1:
-    case 0xfffe:    // MS PCM, Extended
         if (bytes >= 1 && bytes <= 4)
             mp_set_pcm_codec(c, bytes > 1, false, bytes * 8, false);
         break;

demux/demux_mkv.c

@@ -1549,12 +1549,12 @@ static int demux_mkv_open_audio(demuxer_t *demuxer, mkv_track_t *track)
 
     unsigned char *extradata = track->private_data;
     unsigned int extradata_len = track->private_size;
-    uint64_t chmask = 0;
 
     if (!track->a_osfreq)
         track->a_osfreq = track->a_sfreq;
     sh_a->bits_per_coded_sample = track->a_bps ? track->a_bps : 16;
     sh_a->samplerate = (uint32_t) track->a_osfreq;
+    mp_chmap_set_unknown(&sh_a->channels, track->a_channels);
 
     for (int i = 0; mkv_audio_tags[i][0]; i++) {
         if (!strcmp(mkv_audio_tags[i][0], track->codec_id)) {
@@ -1581,10 +1581,11 @@ static int demux_mkv_open_audio(demuxer_t *demuxer, mkv_track_t *track)
         extradata = track->private_data + 18;
         extradata_len = track->private_size - 18;
         sh_a->bits_per_coded_sample = track->a_bps;
+        sh_a->extradata = extradata;
+        sh_a->extradata_size = extradata_len;
         mp_set_codec_from_tag(sh_a);
-        // WAVEFORMATEXTENSIBLE.dwChannelMask
-        if (sh_a->codec_tag == 0xfffe && extradata_len >= 6)
-            chmask = AV_RL32(extradata + 2);
+        extradata = sh_a->extradata;
+        extradata_len = sh_a->extradata_size;
     } else if (!strcmp(track->codec_id, "A_PCM/INT/LIT")) {
         bool sign = sh_a->bits_per_coded_sample > 8;
         mp_set_pcm_codec(sh_a, sign, false, sh_a->bits_per_coded_sample, false);
@@ -1704,10 +1705,6 @@ static int demux_mkv_open_audio(demuxer_t *demuxer, mkv_track_t *track)
     if (!sh_a->codec)
         goto error;
 
-    mp_chmap_from_waveext(&sh_a->channels, chmask);
-    if (sh_a->channels.num != track->a_channels)
-        mp_chmap_set_unknown(&sh_a->channels, track->a_channels);
-
     const char *codec = sh_a->codec;
     if (!strcmp(codec, "mp2") || !strcmp(codec, "mp3") ||
         !strcmp(codec, "truehd"))