diff --git a/player/command.c b/player/command.c
index 712c081d04..e81874010b 100644
--- a/player/command.c
+++ b/player/command.c
@@ -5445,6 +5445,14 @@ static void cmd_sub_step_seek(void *p)
                     track_ind == 0 ? "sub-delay" : "secondary-sub-delay",
                     cmd->on_osd);
             } else {
+                // We can easily seek/step to the wrong subtitle line (because
+                // video frame PTS and sub PTS rarely match exactly).
+                // sub/sd_ass.c adds SUB_SEEK_OFFSET as a workaround, and we
+                // need an even bigger offset without a video.
+                if (!mpctx->current_track[0][STREAM_VIDEO] ||
+                    mpctx->current_track[0][STREAM_VIDEO]->image) {
+                    a[0] += SUB_SEEK_WITHOUT_VIDEO_OFFSET - SUB_SEEK_OFFSET;
+                }
                 mark_seek(mpctx);
                 queue_seek(mpctx, MPSEEK_ABSOLUTE, a[0], MPSEEK_EXACT,
                            MPSEEK_FLAG_DELAY);
diff --git a/sub/sd.h b/sub/sd.h
index f130b94751..459e8c07e9 100644
--- a/sub/sd.h
+++ b/sub/sd.h
@@ -11,6 +11,7 @@
 #define SUB_GAP_KEEP 0.4
 // slight offset when sub seeking or sub stepping
 #define SUB_SEEK_OFFSET 0.01
+#define SUB_SEEK_WITHOUT_VIDEO_OFFSET 0.1
 
 struct sd {
     struct mpv_global *global;