video: move video filter chain initialization from decoder to player

This should help fixing some issues (like not draining video frames
correctly on reinit), as well as decoupling the decoder, filter chain,
and VO code.

I also wanted to make the hardware video decoding fallback work properly
if software-only video filters are inserted. This currently has the
issue that the fallback is too violent, and throws away a bunch of
demuxer packets needed to restart software decoding properly. But
keeping "backup" packets turned out as too hacky, so I'm not doing this,
at least not yet.

mpvcore/player/command.c

@@ -1261,7 +1261,7 @@ static int mp_property_colormatrix(m_option_t *prop, int action, void *arg,
 
     struct mp_image_params vd_csp = {0};
     if (mpctx->d_video)
-        video_vd_control(mpctx->d_video, VDCTRL_GET_PARAMS, &vd_csp);
+        vd_csp = mpctx->d_video->decoder_output;
 
     char *res = talloc_asprintf(NULL, "%s",
                                 mp_csp_names[opts->requested_colorspace]);
@@ -1295,7 +1295,7 @@ static int mp_property_colormatrix_input_range(m_option_t *prop, int action,
 
     struct mp_image_params vd_csp = {0};
     if (mpctx->d_video)
-        video_vd_control(mpctx->d_video, VDCTRL_GET_PARAMS, &vd_csp);
+        vd_csp = mpctx->d_video->decoder_output;
 
     char *res = talloc_asprintf(NULL, "%s",
                                 mp_csp_levels_names[opts->requested_input_range]);

mpvcore/player/mp_core.h

@@ -260,8 +260,6 @@ typedef struct MPContext {
     double last_vo_pts;
     // Video PTS, or audio PTS if video has ended.
     double playback_pts;
-    // Used to determine whether the video filter chain was rebuilt.
-    long last_vf_reconfig_count;
 
     // History of video frames timestamps that were queued in the VO
     // This includes even skipped frames during hr-seek

mpvcore/player/video.c

@@ -38,6 +38,7 @@
 #include "video/hwdec.h"
 #include "video/filter/vf.h"
 #include "video/decode/dec_video.h"
+#include "video/decode/vd.h"
 #include "video/out/vo.h"
 
 #include "mp_core.h"
@@ -81,7 +82,7 @@ int reinit_video_filters(struct MPContext *mpctx)
         return -2;
 
     recreate_video_filters(mpctx);
-    video_reinit_vo(d_video);
+    video_reconfig_filters(d_video, &d_video->decoder_output);
 
     return d_video->vfilter && d_video->vfilter->initialized > 0 ? 0 : -1;
 }
@@ -142,7 +143,6 @@ int reinit_video_chain(struct MPContext *mpctx)
     vo_control(mpctx->video_out, mpctx->paused ? VOCTRL_PAUSE
                                                : VOCTRL_RESUME, NULL);
 
-    mpctx->last_vf_reconfig_count = 0;
     mpctx->restart_playback = true;
     mpctx->sync_audio_to_video = !sh->attached_picture;
     mpctx->delay = 0;
@@ -210,31 +210,53 @@ static bool load_next_vo_frame(struct MPContext *mpctx, bool eof)
 static void init_filter_params(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
+
+    // Note that the filter chain is already initialized. This code might
+    // recreate the chain a second time, which is not very elegant, but allows
+    // us to test whether enabling deinterlacing works with the current video
+    // format and other filters.
+    if (opts->deinterlace >= 0)
+        mp_property_do("deinterlace", M_PROPERTY_SET, &opts->deinterlace, mpctx);
+}
+
+static void reconfig_video(struct MPContext *mpctx,
+                           const struct mp_image_params *params)
+{
     struct dec_video *d_video = mpctx->d_video;
 
-    // Note that the video decoder already initializes the filter chain. This
-    // might recreate the chain a second time, which is not very elegant, but
-    // allows us to test whether enabling deinterlacing works with the current
-    // video format and other filters.
-    if (!d_video->vfilter || d_video->vfilter->initialized != 1)
-        return;
-
-    if (d_video->vf_reconfig_count <= mpctx->last_vf_reconfig_count) {
-        if (opts->deinterlace >= 0) {
-            mp_property_do("deinterlace", M_PROPERTY_SET, &opts->deinterlace,
-                           mpctx);
+    if (!mp_image_params_equals(&d_video->decoder_output, params) ||
+        d_video->vfilter->initialized < 1)
+    {
+        d_video->decoder_output = *params;
+        if (video_reconfig_filters(d_video, params) < 0) {
+            // Most video filters don't work with hardware decoding, so this
+            // might be the reason filter reconfig failed.
+            if (video_vd_control(d_video, VDCTRL_FORCE_HWDEC_FALLBACK, NULL)
+                == CONTROL_OK)
+            {
+                // Fallback active; decoder will return software format next
+                // time. Don't abort video decoding.
+                d_video->vfilter->initialized = 0;
+            }
+            return;
         }
+        if (d_video->vfilter->initialized > 0)
+            init_filter_params(mpctx);
     }
-    // Setting filter params has to be "stable" (no change if params already
-    // set) - checking the reconfig count is just an optimization.
-    mpctx->last_vf_reconfig_count = d_video->vf_reconfig_count;
 }
 
 static void filter_video(struct MPContext *mpctx, struct mp_image *frame)
 {
     struct dec_video *d_video = mpctx->d_video;
 
-    init_filter_params(mpctx);
+    struct mp_image_params params;
+    mp_image_params_from_image(&params, frame);
+    reconfig_video(mpctx, &params);
+
+    if (d_video->vfilter->initialized < 1) {
+        talloc_free(frame);
+        return;
+    }
 
     mp_image_set_params(frame, &d_video->vf_input); // force csp/aspect overrides
     vf_filter_frame(d_video->vfilter, frame);

video/decode/dec_video.c

@@ -114,11 +114,6 @@ int video_get_colors(struct dec_video *d_video, const char *item, int *value)
     return 0;
 }
 
-void video_reinit_vo(struct dec_video *d_video)
-{
-    video_vd_control(d_video, VDCTRL_REINIT_VO, NULL);
-}
-
 void video_uninit(struct dec_video *d_video)
 {
     if (d_video->vd_driver) {
@@ -377,15 +372,13 @@ struct mp_image *video_decode(struct dec_video *d_video,
     return mpi;
 }
 
-int mpcodecs_reconfig_vo(struct dec_video *d_video,
-                         const struct mp_image_params *params)
+int video_reconfig_filters(struct dec_video *d_video,
+                           const struct mp_image_params *params)
 {
     struct MPOpts *opts = d_video->opts;
     struct mp_image_params p = *params;
     struct sh_video *sh = d_video->header->video;
 
-    d_video->vf_reconfig_count++;
-
     mp_msg(MSGT_DECVIDEO, MSGL_V,
            "VIDEO:  %dx%d  %5.3f fps  %5.1f kbps (%4.1f kB/s)\n",
            p.w, p.h, sh->fps, sh->i_bps * 0.008,

video/decode/dec_video.h

@@ -32,13 +32,14 @@ struct dec_video {
     struct MPOpts *opts;
     struct vf_chain *vfilter;  // video filter chain
     const struct vd_functions *vd_driver;
-    long vf_reconfig_count; // incremented each mpcodecs_reconfig_vo() call
-    struct mp_image_params vf_input; // video filter input params
     struct mp_hwdec_info hwdec_info; // video output hwdec handles
     struct sh_stream *header;
 
     char *decoder_desc;
 
+    struct mp_image_params decoder_output; // last output of the decoder
+    struct mp_image_params vf_input; // video filter input params
+
     void *priv; // for free use by vd_driver
 
     // Last PTS from decoder (set with each vd_driver->decode() call)
@@ -89,7 +90,9 @@ struct mp_image *video_decode(struct dec_video *d_video,
 int video_get_colors(struct dec_video *d_video, const char *item, int *value);
 int video_set_colors(struct dec_video *d_video, const char *item, int value);
 void video_reset_decoding(struct dec_video *d_video);
-void video_reinit_vo(struct dec_video *d_video);
 int video_vd_control(struct dec_video *d_video, int cmd, void *arg);
 
+int video_reconfig_filters(struct dec_video *d_video,
+                           const struct mp_image_params *params);
+
 #endif /* MPLAYER_DEC_VIDEO_H */

video/decode/lavc.h

@@ -30,8 +30,6 @@ typedef struct lavc_ctx {
     enum AVPixelFormat pix_fmt;
     int do_hw_dr1;
     int best_csp;
-    struct mp_image_params image_params;
-    struct mp_image_params vo_image_params;
     enum AVDiscard skip_frame;
     const char *software_fallback_decoder;
 

video/decode/vd.h

@@ -42,12 +42,9 @@ typedef struct vd_functions
 extern const vd_functions_t *const mpcodecs_vd_drivers[];
 
 enum vd_ctrl {
-    VDCTRL_GET_PARAMS = 1, // retrieve struct mp_image_params
-    VDCTRL_RESET, // reset decode state after seeking
+    VDCTRL_RESET = 1, // reset decode state after seeking
     VDCTRL_QUERY_UNSEEN_FRAMES, // current decoder lag
-    VDCTRL_REINIT_VO, // reinit filter/VO chain
+    VDCTRL_FORCE_HWDEC_FALLBACK, // force software decoding fallback
 };
 
-int mpcodecs_reconfig_vo(struct dec_video *vd, const struct mp_image_params *params);
-
 #endif /* MPLAYER_VD_H */

video/decode/vd_lavc.c

@@ -383,8 +383,6 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
 
     ctx->do_dr1 = ctx->do_hw_dr1 = 0;
     ctx->pix_fmt = AV_PIX_FMT_NONE;
-    ctx->image_params = (struct mp_image_params){0};
-    ctx->vo_image_params = (struct mp_image_params){0};
     ctx->hwdec = hwdec;
     ctx->avctx = avcodec_alloc_context3(lavc_codec);
     AVCodecContext *avctx = ctx->avctx;
@@ -494,7 +492,8 @@ static void uninit(struct dec_video *vd)
     uninit_avctx(vd);
 }
 
-static void update_image_params(struct dec_video *vd, AVFrame *frame)
+static void update_image_params(struct dec_video *vd, AVFrame *frame,
+                                struct mp_image_params *out_params)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
     int width = frame->width;
@@ -514,7 +513,7 @@ static void update_image_params(struct dec_video *vd, AVFrame *frame)
     int d_w, d_h;
     vf_set_dar(&d_w, &d_h, width, height, aspect);
 
-    ctx->image_params = (struct mp_image_params) {
+    *out_params = (struct mp_image_params) {
         .imgfmt = ctx->best_csp,
         .w = width,
         .h = height,
@@ -741,34 +740,38 @@ static int decode(struct dec_video *vd, struct demux_packet *packet,
     if (!got_picture)
         return 0;
 
-    update_image_params(vd, ctx->pic);
+    struct mp_image_params params;
+    update_image_params(vd, ctx->pic, &params);
     vd->codec_pts = mp_pts_from_av(ctx->pic->pkt_pts, NULL);
     vd->codec_dts = mp_pts_from_av(ctx->pic->pkt_dts, NULL);
 
     // Note: potentially resets ctx->pic as it is transferred to mpi
     struct mp_image *mpi = image_from_decoder(vd);
     assert(mpi->planes[0]);
-    mp_image_set_params(mpi, &ctx->image_params);
+    mp_image_set_params(mpi, &params);
 
     if (ctx->hwdec && ctx->hwdec->process_image)
         mpi = ctx->hwdec->process_image(ctx, mpi);
 
-    struct mp_image_params vo_params;
-    mp_image_params_from_image(&vo_params, mpi);
-
-    if (!mp_image_params_equals(&vo_params, &ctx->vo_image_params)) {
-        mp_image_pool_clear(ctx->non_dr1_pool);
-        if (mpcodecs_reconfig_vo(vd, &vo_params) < 0) {
-            talloc_free(mpi);
-            return -1;
-        }
-        ctx->vo_image_params = vo_params;
-    }
-
     *out_image = mpi;
     return 1;
 }
 
+static int force_fallback(struct dec_video *vd)
+{
+    vd_ffmpeg_ctx *ctx = vd->priv;
+    if (ctx->software_fallback_decoder) {
+        uninit_avctx(vd);
+        mp_tmsg(MSGT_DECVIDEO, MSGL_ERR, "Error using hardware "
+                "decoding, falling back to software decoding.\n");
+        const char *decoder = ctx->software_fallback_decoder;
+        ctx->software_fallback_decoder = NULL;
+        init_avctx(vd, decoder, NULL);
+        return ctx->avctx ? CONTROL_OK : CONTROL_ERROR;
+    }
+    return CONTROL_FALSE;
+}
+
 static struct mp_image *decode_with_fallback(struct dec_video *vd,
                                 struct demux_packet *packet, int flags)
 {
@@ -778,27 +781,13 @@ static struct mp_image *decode_with_fallback(struct dec_video *vd,
 
     struct mp_image *mpi = NULL;
     int res = decode(vd, packet, flags, &mpi);
-    if (res >= 0)
-        return mpi;
-
-    // Failed hardware decoding? Try again in software.
-    if (ctx->software_fallback_decoder) {
-        uninit_avctx(vd);
-        mp_tmsg(MSGT_DECVIDEO, MSGL_ERR, "Error using hardware "
-                "decoding, falling back to software decoding.\n");
-        const char *decoder = ctx->software_fallback_decoder;
-        ctx->software_fallback_decoder = NULL;
-        init_avctx(vd, decoder, NULL);
-        if (ctx->avctx) {
-            mpi = NULL;
-            if (vd->vfilter && vd->vfilter->initialized < 0)
-                vd->vfilter->initialized = 0;
+    if (res < 0) {
+        // Failed hardware decoding? Try again in software.
+        if (force_fallback(vd) == CONTROL_OK)
             decode(vd, packet, flags, &mpi);
-            return mpi;
-        }
     }
 
-    return NULL;
+    return mpi;
 }
 
 static int control(struct dec_video *vd, int cmd, void *arg)
@@ -816,13 +805,8 @@ static int control(struct dec_video *vd, int cmd, void *arg)
             delay += avctx->thread_count - 1;
         *(int *)arg = delay;
         return CONTROL_TRUE;
-    case VDCTRL_REINIT_VO:
-        if (ctx->vo_image_params.imgfmt)
-            mpcodecs_reconfig_vo(vd, &ctx->vo_image_params);
-        return true;
-    case VDCTRL_GET_PARAMS:
-        *(struct mp_image_params *)arg = ctx->vo_image_params;
-        return ctx->vo_image_params.imgfmt ? true : CONTROL_NA;
+    case VDCTRL_FORCE_HWDEC_FALLBACK:
+        return force_fallback(vd);
     }
     return CONTROL_UNKNOWN;
 }