diff --git a/common/playlist.h b/common/playlist.h
index f6111604f3..c5b82781e6 100644
--- a/common/playlist.h
+++ b/common/playlist.h
@@ -43,11 +43,12 @@ struct playlist_entry {
     // Used for unshuffling: the pl_index before it was shuffled. -1 => unknown.
     int original_index;
 
-    // Set to true if playback didn't seem to work, or if the file could be
-    // played only for a very short time. This is used to make playlist
-    // navigation just work in case the user has unplayable files in the
-    // playlist.
-    bool playback_short : 1;
+    // Set to true if this playlist entry was selected while trying to go backwards
+    // in the playlist. If this is true and the playlist entry fails to play later,
+    // then mpv tries to go to the next previous entry. This flag is always cleared
+    // regardless if the attempt was successful or not.
+    bool playlist_prev_attempt : 1;
+
     // Set to true if not at least 1 frame (audio or video) could be played.
     bool init_failed : 1;
     // Entry was removed with playlist_remove (etc.), but not deallocated.
diff --git a/player/loadfile.c b/player/loadfile.c
index 745a19a697..3fcc7b633d 100644
--- a/player/loadfile.c
+++ b/player/loadfile.c
@@ -1567,7 +1567,6 @@ static void load_external_opts(struct MPContext *mpctx)
 static void play_current_file(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
-    double playback_start = -1e100;
 
     assert(mpctx->stop_play);
     mpctx->stop_play = 0;
@@ -1787,6 +1786,7 @@ static void play_current_file(struct MPContext *mpctx)
     MP_VERBOSE(mpctx, "Starting playback...\n");
 
     mpctx->playback_initialized = true;
+    mpctx->playing->playlist_prev_attempt = false;
     mp_notify(mpctx, MPV_EVENT_FILE_LOADED, NULL);
     update_screensaver_state(mpctx);
     clear_playlist_paths(mpctx);
@@ -1828,7 +1828,6 @@ static void play_current_file(struct MPContext *mpctx)
 
     update_internal_pause_state(mpctx);
 
-    playback_start = mp_time_sec();
     mpctx->error_playing = 0;
     mpctx->in_playloop = true;
     while (!mpctx->stop_play)
@@ -1885,6 +1884,7 @@ terminate_playback:
 
     bool nothing_played = !mpctx->shown_aframes && !mpctx->shown_vframes &&
                           mpctx->error_playing <= 0;
+    bool playlist_prev_continue = false;
     switch (mpctx->stop_play) {
     case PT_ERROR:
     case AT_END_OF_FILE:
@@ -1900,10 +1900,10 @@ terminate_playback:
             end_event.reason = MPV_END_FILE_REASON_EOF;
         }
         if (mpctx->playing) {
-            // Played/paused for longer than 1 second -> ok
-            mpctx->playing->playback_short =
-                playback_start < 0 || mp_time_sec() - playback_start < 1.0;
             mpctx->playing->init_failed = nothing_played;
+            playlist_prev_continue = mpctx->playing->playlist_prev_attempt &&
+                                     nothing_played;
+            mpctx->playing->playlist_prev_attempt = false;
         }
         break;
     }
@@ -1939,6 +1939,14 @@ terminate_playback:
     assert(mpctx->stop_play);
 
     process_hooks(mpctx, "on_after_end_file");
+
+    if (playlist_prev_continue) {
+        struct playlist_entry *e = mp_next_file(mpctx, -1, false, true);
+        if (e) {
+            mp_set_playlist_entry(mpctx, e);
+            play_current_file(mpctx);
+        }
+    }
 }
 
 // Determine the next file to play. Note that if this function returns non-NULL,
@@ -1951,12 +1959,9 @@ struct playlist_entry *mp_next_file(struct MPContext *mpctx, int direction,
 {
     struct playlist_entry *next = playlist_get_next(mpctx->playlist, direction);
     if (next && direction < 0 && !force) {
-        // Don't jump to files that would immediately go to next file anyway
-        while (next && next->playback_short)
-            next = playlist_entry_get_rel(next, -1);
-        // Always allow jumping to first file
         if (!next && mpctx->opts->loop_times == 1)
             next = playlist_get_first(mpctx->playlist);
+        next->playlist_prev_attempt = true;
     }
     if (!next && mpctx->opts->loop_times != 1) {
         if (direction > 0) {
@@ -1970,9 +1975,6 @@ struct playlist_entry *mp_next_file(struct MPContext *mpctx, int direction,
             }
         } else {
             next = playlist_get_last(mpctx->playlist);
-            // Don't jump to files that would immediately go to next file anyway
-            while (next && next->playback_short)
-                next = playlist_entry_get_rel(next, -1);
         }
         bool ignore_failures = mpctx->opts->loop_times == -2;
         if (!force && next && next->init_failed && !ignore_failures) {