diff --git a/demux/demux.c b/demux/demux.c
index 5b4b3ec4c2..599218b690 100644
--- a/demux/demux.c
+++ b/demux/demux.c
@@ -232,6 +232,7 @@ struct sh_stream *demux_alloc_sh_stream(enum stream_type type)
         .ff_index = -1,     // may be overwritten by demuxer
         .demuxer_id = -1,   // ... same
         .codec = talloc_zero(sh, struct mp_codec_params),
+        .tags = talloc_zero(sh, struct mp_tags),
     };
     sh->codec->type = type;
     return sh;
@@ -276,6 +277,33 @@ void demux_add_sh_stream(struct demuxer *demuxer, struct sh_stream *sh)
     pthread_mutex_unlock(&in->lock);
 }
 
+// Update sh->tags (lazily). This must be called by demuxers which update
+// stream tags after init. (sh->tags can be accessed by the playback thread,
+// which means the demuxer thread cannot write or read it directly.)
+// Before init is finished, sh->tags can still be accessed freely.
+// Ownership of tags goes to the function.
+void demux_set_stream_tags(struct demuxer *demuxer, struct sh_stream *sh,
+                           struct mp_tags *tags)
+{
+    struct demux_internal *in = demuxer->in;
+    assert(demuxer == in->d_thread);
+
+    if (sh->ds) {
+        while (demuxer->num_update_stream_tags <= sh->index) {
+            MP_TARRAY_APPEND(demuxer, demuxer->update_stream_tags,
+                             demuxer->num_update_stream_tags, NULL);
+        }
+        talloc_free(demuxer->update_stream_tags[sh->index]);
+        demuxer->update_stream_tags[sh->index] = talloc_steal(demuxer, tags);
+
+        demux_changed(demuxer, DEMUX_EVENT_METADATA);
+    } else {
+        // not added yet
+        talloc_free(sh->tags);
+        sh->tags = talloc_steal(sh, tags);
+    }
+}
+
 // Return a stream with the given index. Since streams can only be added during
 // the lifetime of the demuxer, it is guaranteed that an index within the valid
 // range [0, demux_get_num_stream()) always returns a valid sh_stream pointer,
@@ -900,17 +928,18 @@ static int decode_float(char *str, float *out)
     return 0;
 }
 
-static int decode_gain(demuxer_t *demuxer, const char *tag, float *out)
+static int decode_gain(struct mp_log *log, struct mp_tags *tags,
+                       const char *tag, float *out)
 {
     char *tag_val = NULL;
     float dec_val;
 
-    tag_val = mp_tags_get_str(demuxer->metadata, tag);
+    tag_val = mp_tags_get_str(tags, tag);
     if (!tag_val)
         return -1;
 
     if (decode_float(tag_val, &dec_val)) {
-        mp_msg(demuxer->log, MSGL_ERR, "Invalid replaygain value\n");
+        mp_msg(log, MSGL_ERR, "Invalid replaygain value\n");
         return -1;
     }
 
@@ -918,14 +947,15 @@ static int decode_gain(demuxer_t *demuxer, const char *tag, float *out)
     return 0;
 }
 
-static int decode_peak(demuxer_t *demuxer, const char *tag, float *out)
+static int decode_peak(struct mp_log *log, struct mp_tags *tags,
+                       const char *tag, float *out)
 {
     char *tag_val = NULL;
     float dec_val;
 
     *out = 1.0;
 
-    tag_val = mp_tags_get_str(demuxer->metadata, tag);
+    tag_val = mp_tags_get_str(tags, tag);
     if (!tag_val)
         return 0;
 
@@ -939,41 +969,50 @@ static int decode_peak(demuxer_t *demuxer, const char *tag, float *out)
     return 0;
 }
 
-static void apply_replaygain(demuxer_t *demuxer, struct replaygain_data *rg)
+static struct replaygain_data *decode_rgain(struct mp_log *log,
+                                            struct mp_tags *tags)
+{
+    struct replaygain_data rg = {0};
+
+    if (!decode_gain(log, tags, "REPLAYGAIN_TRACK_GAIN", &rg.track_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_TRACK_PEAK", &rg.track_peak) &&
+        !decode_gain(log, tags, "REPLAYGAIN_ALBUM_GAIN", &rg.album_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_ALBUM_PEAK", &rg.album_peak))
+    {
+        return talloc_memdup(NULL, &rg, sizeof(rg));
+    }
+
+    if (!decode_gain(log, tags, "REPLAYGAIN_GAIN", &rg.track_gain) &&
+        !decode_peak(log, tags, "REPLAYGAIN_PEAK", &rg.track_peak))
+    {
+        rg.album_gain = rg.track_gain;
+        rg.album_peak = rg.track_peak;
+        return talloc_memdup(NULL, &rg, sizeof(rg));
+    }
+
+    return NULL;
+}
+
+static void demux_update_replaygain(demuxer_t *demuxer)
 {
     struct demux_internal *in = demuxer->in;
     for (int n = 0; n < in->num_streams; n++) {
         struct sh_stream *sh = in->streams[n];
         if (sh->type == STREAM_AUDIO && !sh->codec->replaygain_data) {
-            MP_VERBOSE(demuxer, "Replaygain: Track=%f/%f Album=%f/%f\n",
-                       rg->track_gain, rg->track_peak,
-                       rg->album_gain, rg->album_peak);
-            sh->codec->replaygain_data = talloc_memdup(in, rg, sizeof(*rg));
+            struct replaygain_data *rg = decode_rgain(demuxer->log, sh->tags);
+            if (!rg)
+                rg = decode_rgain(demuxer->log, demuxer->metadata);
+            if (rg) {
+                MP_VERBOSE(demuxer, "Replaygain/%d: Track=%f/%f Album=%f/%f\n",
+                           sh->index,
+                           rg->track_gain, rg->track_peak,
+                           rg->album_gain, rg->album_peak);
+                sh->codec->replaygain_data = talloc_steal(in, rg);
+            }
         }
     }
 }
 
-static void demux_export_replaygain(demuxer_t *demuxer)
-{
-    struct replaygain_data rg = {0};
-
-    if (!decode_gain(demuxer, "REPLAYGAIN_TRACK_GAIN", &rg.track_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_TRACK_PEAK", &rg.track_peak) &&
-        !decode_gain(demuxer, "REPLAYGAIN_ALBUM_GAIN", &rg.album_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_ALBUM_PEAK", &rg.album_peak))
-    {
-        apply_replaygain(demuxer, &rg);
-    }
-
-    if (!decode_gain(demuxer, "REPLAYGAIN_GAIN", &rg.track_gain) &&
-        !decode_peak(demuxer, "REPLAYGAIN_PEAK", &rg.track_peak))
-    {
-        rg.album_gain = rg.track_gain;
-        rg.album_peak = rg.track_peak;
-        apply_replaygain(demuxer, &rg);
-    }
-}
-
 // Copy all fields from src to dst, depending on event flags.
 static void demux_copy(struct demuxer *dst, struct demuxer *src)
 {
@@ -998,10 +1037,25 @@ static void demux_copy(struct demuxer *dst, struct demuxer *src)
         dst->start_time = src->start_time;
         dst->priv = src->priv;
     }
+
     if (src->events & DEMUX_EVENT_METADATA) {
         talloc_free(dst->metadata);
         dst->metadata = mp_tags_dup(dst, src->metadata);
+
+        if (dst->num_update_stream_tags != src->num_update_stream_tags) {
+            talloc_free(dst->update_stream_tags);
+            dst->update_stream_tags =
+                talloc_zero_array(dst, struct mp_tags *, dst->num_update_stream_tags);
+            dst->num_update_stream_tags = src->num_update_stream_tags;
+        }
+        for (int n = 0; n < dst->num_update_stream_tags; n++) {
+            talloc_free(dst->update_stream_tags[n]);
+            dst->update_stream_tags[n] =
+                talloc_steal(dst->update_stream_tags, src->update_stream_tags[n]);
+            src->update_stream_tags[n] = NULL;
+        }
     }
+
     dst->events |= src->events;
     src->events = 0;
 }
@@ -1023,8 +1077,6 @@ void demux_changed(demuxer_t *demuxer, int events)
 
     if (demuxer->events & DEMUX_EVENT_INIT)
         demuxer_sort_chapters(demuxer);
-    if (demuxer->events & (DEMUX_EVENT_METADATA | DEMUX_EVENT_STREAMS))
-        demux_export_replaygain(demuxer);
 
     demux_copy(in->d_buffer, demuxer);
 
@@ -1047,8 +1099,28 @@ void demux_update(demuxer_t *demuxer)
     demux_copy(demuxer, in->d_buffer);
     demuxer->events |= in->events;
     in->events = 0;
-    if (in->stream_metadata && (demuxer->events & DEMUX_EVENT_METADATA))
-        mp_tags_merge(demuxer->metadata, in->stream_metadata);
+    if (demuxer->events & DEMUX_EVENT_METADATA) {
+        int num_streams = MPMIN(in->num_streams, demuxer->num_update_stream_tags);
+        for (int n = 0; n < num_streams; n++) {
+            struct mp_tags *tags = demuxer->update_stream_tags[n];
+            demuxer->update_stream_tags[n] = NULL;
+            if (tags) {
+                struct sh_stream *sh = in->streams[n];
+                talloc_free(sh->tags);
+                sh->tags = talloc_steal(sh, tags);
+            }
+        }
+
+        // Often useful audio-only files, which have metadata in the audio track
+        // metadata instead of the main metadata (especially OGG).
+        if (in->num_streams == 1)
+            mp_tags_merge(demuxer->metadata, in->streams[0]->tags);
+
+        if (in->stream_metadata)
+            mp_tags_merge(demuxer->metadata, in->stream_metadata);
+    }
+    if (demuxer->events & (DEMUX_EVENT_METADATA | DEMUX_EVENT_STREAMS))
+        demux_update_replaygain(demuxer);
     pthread_mutex_unlock(&in->lock);
 }
 
diff --git a/demux/demux.h b/demux/demux.h
index 8470047f9d..07803d2838 100644
--- a/demux/demux.h
+++ b/demux/demux.h
@@ -210,7 +210,10 @@ typedef struct demuxer {
     struct mp_log *log, *glog;
     struct demuxer_params *params;
 
-    struct demux_internal *in; // internal to demux.c
+    // internal to demux.c
+    struct demux_internal *in;
+    struct mp_tags **update_stream_tags;
+    int num_update_stream_tags;
 
     // Since the demuxer can run in its own thread, and the stream is not
     // thread-safe, only the demuxer is allowed to access the stream directly.
@@ -273,6 +276,8 @@ int demuxer_add_attachment(struct demuxer *demuxer, char *name,
                            char *type, void *data, size_t data_size);
 int demuxer_add_chapter(demuxer_t *demuxer, char *name,
                         double pts, uint64_t demuxer_id);
+void demux_set_stream_tags(struct demuxer *demuxer, struct sh_stream *sh,
+                           struct mp_tags *tags);
 
 double demuxer_get_time_length(struct demuxer *demuxer);
 
diff --git a/demux/demux_lavf.c b/demux/demux_lavf.c
index a38fb60435..e28ebd0c23 100644
--- a/demux/demux_lavf.c
+++ b/demux/demux_lavf.c
@@ -168,7 +168,6 @@ typedef struct lavf_priv {
     int num_streams;
     int cur_program;
     char *mime_type;
-    bool merge_track_metadata;
     double seek_delay;
 } lavf_priv_t;
 
@@ -502,7 +501,7 @@ static void select_tracks(struct demuxer *demuxer, int start)
     }
 }
 
-static void export_replaygain(demuxer_t *demuxer, struct mp_codec_params *c,
+static void export_replaygain(demuxer_t *demuxer, struct sh_stream *sh,
                               AVStream *st)
 {
     for (int i = 0; i < st->nb_side_data; i++) {
@@ -528,7 +527,10 @@ static void export_replaygain(demuxer_t *demuxer, struct mp_codec_params *c,
         rgain->album_peak = (av_rgain->album_peak != 0.0) ?
             av_rgain->album_peak / 100000.0f : 1.0;
 
-        c->replaygain_data = rgain;
+        // This must be run only before the stream was added, otherwise there
+        // will be race conditions with accesses from the user thread.
+        assert(!sh->ds);
+        sh->codec->replaygain_data = rgain;
     }
 }
 
@@ -575,7 +577,7 @@ static void handle_new_stream(demuxer_t *demuxer, int i)
             delay = lavc_delay / (double)codec->sample_rate;
         priv->seek_delay = MPMAX(priv->seek_delay, delay);
 
-        export_replaygain(demuxer, sh->codec, st);
+        export_replaygain(demuxer, sh, st);
 
         break;
     }
@@ -681,6 +683,7 @@ static void handle_new_stream(demuxer_t *demuxer, int i)
         if (!sh->title && sh->hls_bitrate > 0)
             sh->title = talloc_asprintf(sh, "bitrate %d", sh->hls_bitrate);
         sh->missing_timestamps = !!(priv->avif_flags & AVFMT_NOTIMESTAMPS);
+        mp_tags_copy_from_av_dictionary(sh->tags, st->metadata);
         demux_add_sh_stream(demuxer, sh);
     }
 
@@ -703,14 +706,14 @@ static void update_metadata(demuxer_t *demuxer, AVPacket *pkt)
         priv->avfc->event_flags = 0;
         demux_changed(demuxer, DEMUX_EVENT_METADATA);
     }
-    if (priv->merge_track_metadata) {
-        for (int n = 0; n < priv->num_streams; n++) {
-            AVStream *st = priv->streams[n] ? priv->avfc->streams[n] : NULL;
-            if (st && st->event_flags & AVSTREAM_EVENT_FLAG_METADATA_UPDATED) {
-                mp_tags_copy_from_av_dictionary(demuxer->metadata, st->metadata);
-                st->event_flags = 0;
-                demux_changed(demuxer, DEMUX_EVENT_METADATA);
-            }
+
+    for (int n = 0; n < priv->num_streams; n++) {
+        AVStream *st = priv->streams[n] ? priv->avfc->streams[n] : NULL;
+        if (st && st->event_flags & AVSTREAM_EVENT_FLAG_METADATA_UPDATED) {
+            st->event_flags = 0;
+            struct mp_tags *tags = talloc_zero(NULL, struct mp_tags);
+            mp_tags_copy_from_av_dictionary(tags, st->metadata);
+            demux_set_stream_tags(demuxer, priv->streams[n], tags);
         }
     }
 }
@@ -846,16 +849,6 @@ static int demux_open_lavf(demuxer_t *demuxer, enum demux_check check)
 
     add_new_streams(demuxer);
 
-    // Often useful with OGG audio-only files, which have metadata in the audio
-    // track metadata instead of the main metadata.
-    if (demux_get_num_stream(demuxer) == 1) {
-        priv->merge_track_metadata = true;
-        for (int n = 0; n < priv->num_streams; n++) {
-            if (priv->streams[n])
-                mp_tags_copy_from_av_dictionary(demuxer->metadata, avfc->streams[n]->metadata);
-        }
-    }
-
     mp_tags_copy_from_av_dictionary(demuxer->metadata, avfc->metadata);
     update_metadata(demuxer, NULL);
 
diff --git a/demux/demux_mkv.c b/demux/demux_mkv.c
index 6f45b5a5f2..da15c0fc0f 100644
--- a/demux/demux_mkv.c
+++ b/demux/demux_mkv.c
@@ -93,6 +93,7 @@ typedef struct mkv_content_encoding {
 
 typedef struct mkv_track {
     int tnum;
+    uint64_t uid;
     char *name;
     struct sh_stream *stream;
 
@@ -595,6 +596,7 @@ static void parse_trackentry(struct demuxer *demuxer,
     } else {
         MP_ERR(demuxer, "Missing track number!\n");
     }
+    track->uid = entry->track_uid;
 
     if (entry->name) {
         track->name = talloc_strdup(track, entry->name);
@@ -986,9 +988,6 @@ static void process_tags(demuxer_t *demuxer)
 
     for (int i = 0; i < tags->n_tag; i++) {
         struct ebml_tag tag = tags->tag[i];
-        if (tag.targets.target_track_uid  || tag.targets.target_attachment_uid)
-            continue;
-
         struct mp_tags *dst = NULL;
 
         if (tag.targets.target_chapter_uid) {
@@ -1009,6 +1008,19 @@ static void process_tags(demuxer_t *demuxer)
                     break;
                 }
             }
+        } else if (tag.targets.target_track_uid) {
+            for (int n = 0; n < mkv_d->num_tracks; n++) {
+                if (mkv_d->tracks[n]->uid ==
+                    tag.targets.target_track_uid)
+                {
+                    struct sh_stream *sh = mkv_d->tracks[n]->stream;
+                    if (sh)
+                        dst = sh->tags;
+                    break;
+                }
+            }
+        } else if (tag.targets.target_attachment_uid) {
+            /* ignore */
         } else {
             dst = demuxer->metadata;
         }
@@ -1924,9 +1936,9 @@ static int demux_mkv_open(demuxer_t *demuxer, enum demux_check check)
 
     MP_VERBOSE(demuxer, "All headers are parsed!\n");
 
-    process_tags(demuxer);
     display_create_tracks(demuxer);
     add_coverart(demuxer);
+    process_tags(demuxer);
 
     probe_first_timestamp(demuxer);
     if (opts->demux_mkv->probe_duration)
diff --git a/demux/stheader.h b/demux/stheader.h
index 77d0eb1da3..f9d564c230 100644
--- a/demux/stheader.h
+++ b/demux/stheader.h
@@ -46,6 +46,8 @@ struct sh_stream {
     bool forced_track;          // container forced track flag
     int hls_bitrate;
 
+    struct mp_tags *tags;
+
     bool missing_timestamps;
 
     // stream is a picture (such as album art)