diff --git a/libavfilter/vf_paletteuse.c b/libavfilter/vf_paletteuse.c
index 1225a660bb..dece05a0f8 100644
--- a/libavfilter/vf_paletteuse.c
+++ b/libavfilter/vf_paletteuse.c
@@ -1003,6 +1003,7 @@ static av_cold int init(AVFilterContext *ctx)
 {
     PaletteUseContext *s = ctx->priv;
     s->dinput.repeatlast = 1; // only 1 frame in the palette
+    s->dinput.skip_initial_unpaired = 1;
     s->dinput.process    = load_apply_palette;
 
     s->set_frame = set_frame_lut[s->color_search_method][s->dither];