diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 68f415035c..cf94e32ce9 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -3335,6 +3335,12 @@ static int mov_write_mfhd_tag(AVIOContext *pb, MOVMuxContext *mov)
     return 0;
 }
 
+static uint32_t get_sample_flags(MOVTrack *track, MOVIentry *entry)
+{
+    return entry->flags & MOV_SYNC_SAMPLE ? MOV_FRAG_SAMPLE_FLAG_DEPENDS_NO :
+           (MOV_FRAG_SAMPLE_FLAG_DEPENDS_YES | MOV_FRAG_SAMPLE_FLAG_IS_NON_SYNC);
+}
+
 static int mov_write_tfhd_tag(AVIOContext *pb, MOVMuxContext *mov,
                               MOVTrack *track, int64_t moof_offset)
 {
@@ -3380,22 +3386,21 @@ static int mov_write_tfhd_tag(AVIOContext *pb, MOVMuxContext *mov,
         track->default_size = -1;
 
     if (flags & MOV_TFHD_DEFAULT_FLAGS) {
-        track->default_sample_flags =
-            track->enc->codec_type == AVMEDIA_TYPE_VIDEO ?
-            (MOV_FRAG_SAMPLE_FLAG_DEPENDS_YES | MOV_FRAG_SAMPLE_FLAG_IS_NON_SYNC) :
-            MOV_FRAG_SAMPLE_FLAG_DEPENDS_NO;
+        /* Set the default flags based on the second sample, if available.
+         * If the first sample is different, that can be signaled via a separate field. */
+        if (track->entry > 1)
+            track->default_sample_flags = get_sample_flags(track, &track->cluster[1]);
+        else
+            track->default_sample_flags =
+                track->enc->codec_type == AVMEDIA_TYPE_VIDEO ?
+                (MOV_FRAG_SAMPLE_FLAG_DEPENDS_YES | MOV_FRAG_SAMPLE_FLAG_IS_NON_SYNC) :
+                MOV_FRAG_SAMPLE_FLAG_DEPENDS_NO;
         avio_wb32(pb, track->default_sample_flags);
     }
 
     return update_size(pb, pos);
 }
 
-static uint32_t get_sample_flags(MOVTrack *track, MOVIentry *entry)
-{
-    return entry->flags & MOV_SYNC_SAMPLE ? MOV_FRAG_SAMPLE_FLAG_DEPENDS_NO :
-           (MOV_FRAG_SAMPLE_FLAG_DEPENDS_YES | MOV_FRAG_SAMPLE_FLAG_IS_NON_SYNC);
-}
-
 static int mov_write_trun_tag(AVIOContext *pb, MOVMuxContext *mov,
                               MOVTrack *track, int moof_size)
 {
diff --git a/tests/ref/fate/sub-movtextenc b/tests/ref/fate/sub-movtextenc
index e74b8f2b35..a44065c846 100644
--- a/tests/ref/fate/sub-movtextenc
+++ b/tests/ref/fate/sub-movtextenc
@@ -1 +1 @@
-bb762c178bd8c437a9101c748c1ccb4d
+930cfa0d91434f46a8a197a2165a734a