diff --git a/libavformat/internal.h b/libavformat/internal.h
index bffb8e66ff..f24c68703f 100644
--- a/libavformat/internal.h
+++ b/libavformat/internal.h
@@ -82,6 +82,17 @@ typedef struct FFFormatContext {
      */
     int nb_interleaved_streams;
 
+    /**
+     * Whether the timestamp shift offset has already been determined.
+     * -1: disabled, 0: not yet determined, 1: determined.
+     */
+    enum {
+        AVOID_NEGATIVE_TS_DISABLED = -1,
+        AVOID_NEGATIVE_TS_UNKNOWN  = 0,
+        AVOID_NEGATIVE_TS_KNOWN    = 1,
+    } avoid_negative_ts_status;
+#define AVOID_NEGATIVE_TS_ENABLED(status) ((status) >= 0)
+
     /**
      * The interleavement function in use. Always set for muxers.
      */
@@ -135,18 +146,6 @@ typedef struct FFFormatContext {
      */
     int raw_packet_buffer_size;
 
-    /**
-     * Offset to remap timestamps to be non-negative.
-     * Expressed in timebase units.
-     * @see AVStream.mux_ts_offset
-     */
-    int64_t offset;
-
-    /**
-     * Timebase for the timestamp offset.
-     */
-    AVRational offset_timebase;
-
 #if FF_API_COMPUTE_PKT_FIELDS2
     int missing_ts_warning;
 #endif
diff --git a/libavformat/mux.c b/libavformat/mux.c
index a1917878a5..0810b674a7 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -388,6 +388,8 @@ fail:
 
 static int init_pts(AVFormatContext *s)
 {
+    FFFormatContext *const si = ffformatcontext(s);
+
     /* init PTS generation */
     for (unsigned i = 0; i < s->nb_streams; i++) {
         AVStream *const st = s->streams[i];
@@ -418,13 +420,16 @@ static int init_pts(AVFormatContext *s)
         }
     }
 
+    si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_UNKNOWN;
     if (s->avoid_negative_ts < 0) {
         av_assert2(s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_AUTO);
         if (s->oformat->flags & (AVFMT_TS_NEGATIVE | AVFMT_NOTIMESTAMPS)) {
             s->avoid_negative_ts = AVFMT_AVOID_NEG_TS_DISABLED;
+            si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_DISABLED;
         } else
             s->avoid_negative_ts = AVFMT_AVOID_NEG_TS_MAKE_NON_NEGATIVE;
-    }
+    } else if (s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_DISABLED)
+        si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_DISABLED;
 
     return 0;
 }
@@ -638,6 +643,64 @@ static void guess_pkt_duration(AVFormatContext *s, AVStream *st, AVPacket *pkt)
     }
 }
 
+static void handle_avoid_negative_ts(FFFormatContext *si, FFStream *sti,
+                                     AVPacket *pkt)
+{
+    AVFormatContext *const s = &si->pub;
+    int64_t offset;
+
+    if (!AVOID_NEGATIVE_TS_ENABLED(si->avoid_negative_ts_status))
+        return;
+
+    if (si->avoid_negative_ts_status == AVOID_NEGATIVE_TS_UNKNOWN) {
+        int use_pts = si->avoid_negative_ts_use_pts;
+        int64_t ts = use_pts ? pkt->pts : pkt->dts;
+
+        if (ts == AV_NOPTS_VALUE)
+            return;
+        if (ts < 0 ||
+            ts > 0 && s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO) {
+            for (unsigned i = 0; i < s->nb_streams; i++) {
+                AVStream *const st2  = s->streams[i];
+                FFStream *const sti2 = ffstream(st2);
+                sti2->mux_ts_offset = av_rescale_q_rnd(-ts,
+                                                       sti->pub.time_base,
+                                                       st2->time_base,
+                                                       AV_ROUND_UP);
+            }
+        }
+        si->avoid_negative_ts_status = AVOID_NEGATIVE_TS_KNOWN;
+    }
+
+    offset = sti->mux_ts_offset;
+
+    if (pkt->dts != AV_NOPTS_VALUE)
+        pkt->dts += offset;
+    if (pkt->pts != AV_NOPTS_VALUE)
+        pkt->pts += offset;
+
+    if (si->avoid_negative_ts_use_pts) {
+        if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < 0) {
+            av_log(s, AV_LOG_WARNING, "failed to avoid negative "
+                   "pts %s in stream %d.\n"
+                   "Try -avoid_negative_ts 1 as a possible workaround.\n",
+                   av_ts2str(pkt->pts),
+                   pkt->stream_index
+            );
+        }
+    } else {
+        if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < 0) {
+            av_log(s, AV_LOG_WARNING,
+                   "Packets poorly interleaved, failed to avoid negative "
+                   "timestamp %s in stream %d.\n"
+                   "Try -max_interleave_delta 0 as a possible workaround.\n",
+                   av_ts2str(pkt->dts),
+                   pkt->stream_index
+            );
+        }
+    }
+}
+
 /**
  * Shift timestamps and call muxer; the original pts/dts are not kept.
  *
@@ -663,51 +726,7 @@ static int write_packet(AVFormatContext *s, AVPacket *pkt)
         if (pkt->pts != AV_NOPTS_VALUE)
             pkt->pts += offset;
     }
-
-    if (s->avoid_negative_ts > 0) {
-        int64_t offset = sti->mux_ts_offset;
-        int64_t ts = si->avoid_negative_ts_use_pts ? pkt->pts : pkt->dts;
-
-        if (si->offset == AV_NOPTS_VALUE && ts != AV_NOPTS_VALUE &&
-            (ts < 0 || s->avoid_negative_ts == AVFMT_AVOID_NEG_TS_MAKE_ZERO)) {
-            si->offset = -ts;
-            si->offset_timebase = st->time_base;
-        }
-
-        if (si->offset != AV_NOPTS_VALUE && !offset) {
-            offset = sti->mux_ts_offset =
-                av_rescale_q_rnd(si->offset,
-                                 si->offset_timebase,
-                                 st->time_base,
-                                 AV_ROUND_UP);
-        }
-
-        if (pkt->dts != AV_NOPTS_VALUE)
-            pkt->dts += offset;
-        if (pkt->pts != AV_NOPTS_VALUE)
-            pkt->pts += offset;
-
-        if (si->avoid_negative_ts_use_pts) {
-            if (pkt->pts != AV_NOPTS_VALUE && pkt->pts < 0) {
-                av_log(s, AV_LOG_WARNING, "failed to avoid negative "
-                    "pts %s in stream %d.\n"
-                    "Try -avoid_negative_ts 1 as a possible workaround.\n",
-                    av_ts2str(pkt->pts),
-                    pkt->stream_index
-                );
-            }
-        } else {
-            if (pkt->dts != AV_NOPTS_VALUE && pkt->dts < 0) {
-                av_log(s, AV_LOG_WARNING,
-                    "Packets poorly interleaved, failed to avoid negative "
-                    "timestamp %s in stream %d.\n"
-                    "Try -max_interleave_delta 0 as a possible workaround.\n",
-                    av_ts2str(pkt->dts),
-                    pkt->stream_index
-                );
-            }
-        }
-    }
+    handle_avoid_negative_ts(si, sti, pkt);
 
     if ((pkt->flags & AV_PKT_FLAG_UNCODED_FRAME)) {
         AVFrame **frame = (AVFrame **)pkt->data;
diff --git a/libavformat/options.c b/libavformat/options.c
index 1634388acb..2d55d3ad6e 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -174,7 +174,6 @@ AVFormatContext *avformat_alloc_context(void)
         return NULL;
     }
 
-    si->offset = AV_NOPTS_VALUE;
     si->shortest_end = AV_NOPTS_VALUE;
 
     return s;
diff --git a/tests/fate/matroska.mak b/tests/fate/matroska.mak
index 1a45e8c022..9d47ade08f 100644
--- a/tests/fate/matroska.mak
+++ b/tests/fate/matroska.mak
@@ -101,8 +101,8 @@ fate-matroska-dovi-write-config8: CMD = transcode mov $(TARGET_SAMPLES)/hevc/dv8
 # the first packet (with the overall lowest dts) is a video packet,
 # whereas an audio packet to be muxed later has the overall lowest pts
 # which happens to be negative and therefore needs to be shifted.
-# This is currently buggy (the timestamps of the video frames muxed
-# before the first audio frame are not shifted).
+# This is currently buggy (the timestamps are not shifted properly:
+# the first audio packet has negative pts).
 # (-ss 1.09 ensures that a video frame has the lowest dts of all packets;
 # yet there is an audio packet with the overall lowest pts. output_ts_offset
 # makes the pts of the audio packet, but not the leading video packet negative
diff --git a/tests/ref/fate/matroska-avoid-negative-ts b/tests/ref/fate/matroska-avoid-negative-ts
index a687c8f63c..1b9b2f2786 100644
--- a/tests/ref/fate/matroska-avoid-negative-ts
+++ b/tests/ref/fate/matroska-avoid-negative-ts
@@ -1,4 +1,4 @@
-3349536550047c5c553215003ba2acb7 *tests/data/fate/matroska-avoid-negative-ts.matroska
+90cf5a330659140d47ec11208f525908 *tests/data/fate/matroska-avoid-negative-ts.matroska
 973070 tests/data/fate/matroska-avoid-negative-ts.matroska
 #extradata 0:       22, 0x2885037c
 #tb 0: 1/1000
@@ -12,11 +12,11 @@
 #sample_rate 1: 44100
 #channel_layout 1: 4
 #channel_layout_name 1: mono
-0,        -37,         24,       40,     9156, 0xe5bd034a, S=1,       40
+0,        -37,         43,       40,     9156, 0xe5bd034a, S=1,       40
 1,          0,          0,       26,      417, 0x7198c15e
 0,          3,          3,       40,     1740, 0x29ac4480, F=0x0
-0,         24,        123,       40,     3672, 0x98652013, F=0x0
 1,         26,         26,       26,      417, 0x3c67c32d
+0,         43,        123,       40,     3672, 0x98652013, F=0x0
 1,         52,         52,       26,      417, 0x8c24b1ca
 1,         78,         78,       26,      417, 0x6ee576b7
 0,         83,         83,       40,     2532, 0xa2c42769, F=0x0