video: make vdpau hardware decoding not use slices code path

For some reason, libavcodec abuses the slices rendering code path for
hardware decoding: in that case, the only purpose of the draw callback
is to pass a vdpau video surface object to video output. (It is unclear
to me why this had to use the slices code, instead of just returning an
AVFrame with the required vdpau state.)

Make this code separate within mpv, so that the internal slices code
path is not used for hardware decoding. Pass the vdpau state with
VOCTRL_HWDEC_DECODER_RENDER instead.

Remove the mencoder specific VOCTRLs.

video/decode/vd_lavc.c

@@ -40,6 +40,7 @@
 
 #include "vd.h"
 #include "video/img_format.h"
+#include "video/filter/vf.h"
 #include "demux/stheader.h"
 #include "demux/demux_packet.h"
 #include "core/codec-cfg.h"
@@ -84,6 +85,8 @@ static int get_buffer(AVCodecContext *avctx, AVFrame *pic);
 static void release_buffer(AVCodecContext *avctx, AVFrame *pic);
 static void draw_slice(struct AVCodecContext *s, const AVFrame *src,
                        int offset[4], int y, int type, int height);
+static void draw_slice_hwdec(struct AVCodecContext *s, const AVFrame *src,
+                             int offset[4], int y, int type, int height);
 
 static enum PixelFormat get_format(struct AVCodecContext *avctx,
                                    const enum PixelFormat *pix_fmt);
@@ -195,7 +198,7 @@ static int init(sh_video_t *sh)
         avctx->get_buffer      = get_buffer;
         avctx->release_buffer  = release_buffer;
         avctx->reget_buffer    = get_buffer;
-        avctx->draw_horiz_band = draw_slice;
+        avctx->draw_horiz_band = draw_slice_hwdec;
         if (lavc_codec->capabilities & CODEC_CAP_HWACCEL_VDPAU)
             mp_msg(MSGT_DECVIDEO, MSGL_V, "[VD_FFMPEG] VDPAU hardware "
                    "decoding.\n");
@@ -371,6 +374,16 @@ static void uninit(sh_video_t *sh)
     talloc_free(ctx);
 }
 
+static void draw_slice_hwdec(struct AVCodecContext *s,
+                             const AVFrame *src, int offset[4],
+                             int y, int type, int height)
+{
+    sh_video_t *sh = s->opaque;
+    struct vf_instance *vf = sh->vfilter;
+    void *state_ptr = src->data[0];
+    vf->control(vf, VFCTRL_HWDEC_DECODER_RENDER, state_ptr);
+}
+
 static void draw_slice(struct AVCodecContext *s,
                        const AVFrame *src, int offset[4],
                        int y, int type, int height)
@@ -544,7 +557,7 @@ static int get_buffer(AVCodecContext *avctx, AVFrame *pic)
     } else
         avctx->draw_horiz_band = NULL;
     if (IMGFMT_IS_HWACCEL(mpi->imgfmt))
-        avctx->draw_horiz_band = draw_slice;
+        avctx->draw_horiz_band = draw_slice_hwdec;
 
     pic->data[0] = mpi->planes[0];
     pic->data[1] = mpi->planes[1];

video/filter/vf.h

@@ -100,9 +100,7 @@ struct vf_ctrl_screenshot {
 #define VFCTRL_SET_PP_LEVEL 5       // set postprocessing level
 #define VFCTRL_SET_EQUALIZER 6 // set color options (brightness,contrast etc)
 #define VFCTRL_GET_EQUALIZER 8 // get color options (brightness,contrast etc)
-#define VFCTRL_DUPLICATE_FRAME 11  // For encoding - encode zero-change frame
-#define VFCTRL_SKIP_NEXT_FRAME 12  // For encoding - drop the next frame that passes thru
-#define VFCTRL_FLUSH_FRAMES    13  // For encoding - flush delayed frames
+#define VFCTRL_HWDEC_DECODER_RENDER 9 // vdpau hw decoding
 #define VFCTRL_SCREENSHOT      14  // Take screenshot, arg is vf_ctrl_screenshot
 #define VFCTRL_INIT_OSD        15  // Filter OSD renderer present?
 #define VFCTRL_SET_DEINTERLACE 18  // Set deinterlacing status

video/filter/vf_vo.c

@@ -107,6 +107,8 @@ static int control(struct vf_instance *vf, int request, void *data)
         };
         return vo_control(video_out, VOCTRL_GET_EQUALIZER, &param) == VO_TRUE;
     }
+    case VFCTRL_HWDEC_DECODER_RENDER:
+        return vo_control(video_out, VOCTRL_HWDEC_DECODER_RENDER, data);
     }
     return CONTROL_UNKNOWN;
 }

video/out/vo.h

@@ -54,6 +54,9 @@ enum mp_voctrl {
 
     VOCTRL_START_SLICE,
 
+    /* for vdpau hardware decoding */
+    VOCTRL_HWDEC_DECODER_RENDER,        // pointer to hw state
+
     VOCTRL_NEWFRAME,
     VOCTRL_SKIPFRAME,
     VOCTRL_REDRAW_FRAME,

video/out/vo_vdpau.c

@@ -1252,17 +1252,16 @@ static void flip_page_timed(struct vo *vo, unsigned int pts_us, int duration)
     vc->surface_num = WRAP_ADD(vc->surface_num, 1, vc->num_output_surfaces);
 }
 
-static int draw_slice(struct vo *vo, uint8_t *image[], int stride[], int w,
-                      int h, int x, int y)
+static int decoder_render(struct vo *vo, void *state_ptr)
 {
     struct vdpctx *vc = vo->priv;
     struct vdp_functions *vdp = vc->vdp;
     VdpStatus vdp_st;
+    struct vdpau_render_state *rndr = (struct vdpau_render_state *)state_ptr;
 
     if (handle_preemption(vo) < 0)
         return VO_TRUE;
 
-    struct vdpau_render_state *rndr = (struct vdpau_render_state *)image[0];
     int max_refs = vc->image_format == IMGFMT_VDPAU_H264 ?
         rndr->info.h264.num_ref_frames : 2;
     if (!IMGFMT_IS_VDPAU(vc->image_format))
@@ -1606,6 +1605,8 @@ static int control(struct vo *vo, uint32_t request, void *data)
         return true;
     case VOCTRL_GET_IMAGE:
         return get_image(vo, data);
+    case VOCTRL_HWDEC_DECODER_RENDER:
+        return decoder_render(vo, data);
     case VOCTRL_BORDER:
         vo_x11_border(vo);
         checked_resize(vo);
@@ -1689,7 +1690,6 @@ const struct vo_driver video_out_vdpau = {
     .control = control,
     .draw_image_pts = draw_image,
     .get_buffered_frame = set_next_frame_info,
-    .draw_slice = draw_slice,
     .draw_osd = draw_osd,
     .flip_page_timed = flip_page_timed,
     .check_events = check_events,